From a9b0b977039aee7a5870503cd79624149e45efb2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 27 Oct 2016 22:53:52 +0200 Subject: [PATCH 01/13] Expose Lucenes Ukrainian analyzer Since Lucene 6.2. the UkrainianMorfologikAnalyzer is available through the lucene-analyzers-morfologik jar. This change exposes it to be used as an elasticsearch plugin. --- plugins/analysis-ukrainian/build.gradle | 39 ++ .../licenses/lucene-LICENSE.txt | 475 ++++++++++++++++++ .../licenses/lucene-NOTICE.txt | 191 +++++++ ...lucene-analyzers-morfologik-6.2.0.jar.sha1 | 1 + .../licenses/morfologik-fsa-2.1.0.jar.sha1 | 1 + .../morfologik-stemming-2.1.0.jar.sha1 | 1 + .../analysis/UkrainianAnalyzerProvider.java | 47 ++ .../ukrainian/AnalysisUkrainianPlugin.java | 39 ++ .../SimpleUkrainianAnalyzerTests.java | 56 +++ .../analysis/UkrainianAnalysisTests.java | 43 ++ .../UkrainianClientYamlTestSuiteIT.java | 42 ++ .../test/analysis_ukrainian/10_basic.yaml | 18 + .../test/analysis_ukrainian/20_search.yaml | 32 ++ settings.gradle | 1 + 14 files changed, 986 insertions(+) create mode 100644 plugins/analysis-ukrainian/build.gradle create mode 100644 plugins/analysis-ukrainian/licenses/lucene-LICENSE.txt create mode 100644 plugins/analysis-ukrainian/licenses/lucene-NOTICE.txt create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java create mode 100644 plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java create mode 100644 plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/SimpleUkrainianAnalyzerTests.java create mode 100644 plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java create mode 100644 plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java create mode 100644 plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/10_basic.yaml create mode 100644 plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yaml diff --git a/plugins/analysis-ukrainian/build.gradle b/plugins/analysis-ukrainian/build.gradle new file mode 100644 index 00000000000..b3c5473a2ff --- /dev/null +++ b/plugins/analysis-ukrainian/build.gradle @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +esplugin { + description 'The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch.' + classname 'org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin' +} + +dependencies { + compile "org.apache.lucene:lucene-analyzers-morfologik:${versions.lucene}" + compile "org.carrot2:morfologik-stemming:2.1.0" + compile "org.carrot2:morfologik-fsa:2.1.0" +} + +dependencyLicenses { + mapping from: /lucene-.*/, to: 'lucene' + mapping from: /morfologik-.*/, to: 'lucene' +} + +thirdPartyAudit.excludes = [ + // we don't use the morfologik-fsa polish stemmer + 'morfologik.stemming.polish.PolishStemmer' +] diff --git a/plugins/analysis-ukrainian/licenses/lucene-LICENSE.txt b/plugins/analysis-ukrainian/licenses/lucene-LICENSE.txt new file mode 100644 index 00000000000..28b134f5f8e --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-LICENSE.txt @@ -0,0 +1,475 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from unicode conversion examples available at +http://www.unicode.org/Public/PROGRAMS/CVTUTF. Here is the copyright +from those sources: + +/* + * Copyright 2001-2004 Unicode, Inc. + * + * Disclaimer + * + * This source code is provided as is by Unicode, Inc. No claims are + * made as to fitness for any particular purpose. No warranties of any + * kind are expressed or implied. The recipient agrees to determine + * applicability of information provided. If this file has been + * purchased on magnetic or optical media from Unicode, Inc., the + * sole remedy for any claim will be exchange of defective media + * within 90 days of receipt. + * + * Limitations on Rights to Redistribute This Code + * + * Unicode, Inc. hereby grants the right to freely use the information + * supplied in this file in the creation of products supporting the + * Unicode Standard, and to make copies of this file in any form + * for internal or external distribution as long as this notice + * remains attached. + */ + + +Some code in core/src/java/org/apache/lucene/util/ArrayUtil.java was +derived from Python 2.4.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/2.4.2/license/ + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from Python 3.1.2 sources available at +http://www.python.org. Full license is here: + + http://www.python.org/download/releases/3.1.2/license/ + +Some code in core/src/java/org/apache/lucene/util/automaton was +derived from Brics automaton sources available at +www.brics.dk/automaton/. Here is the copyright from those sources: + +/* + * Copyright (c) 2001-2009 Anders Moeller + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +The levenshtein automata tables in core/src/java/org/apache/lucene/util/automaton +were automatically generated with the moman/finenight FSA package. +Here is the copyright for those sources: + +# Copyright (c) 2010, Jean-Philippe Barrette-LaPierre, +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the "Software"), to deal in the Software without +# restriction, including without limitation the rights to use, +# copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the following +# conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES +# OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +# OTHER DEALINGS IN THE SOFTWARE. + +Some code in core/src/java/org/apache/lucene/util/UnicodeUtil.java was +derived from ICU (http://www.icu-project.org) +The full license is available here: + http://source.icu-project.org/repos/icu/icu/trunk/license.html + +/* + * Copyright (C) 1999-2010, International Business Machines + * Corporation and others. All Rights Reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, and/or sell copies of the + * Software, and to permit persons to whom the Software is furnished to do so, + * provided that the above copyright notice(s) and this permission notice appear + * in all copies of the Software and that both the above copyright notice(s) and + * this permission notice appear in supporting documentation. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT OF THIRD PARTY RIGHTS. + * IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS NOTICE BE + * LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR + * ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER + * IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT + * OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + * + * Except as contained in this notice, the name of a copyright holder shall not + * be used in advertising or otherwise to promote the sale, use or other + * dealings in this Software without prior written authorization of the + * copyright holder. + */ + +The following license applies to the Snowball stemmers: + +Copyright (c) 2001, Dr Martin Porter +Copyright (c) 2002, Richard Boulton +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * Neither the name of the copyright holders nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The following license applies to the KStemmer: + +Copyright © 2003, +Center for Intelligent Information Retrieval, +University of Massachusetts, Amherst. +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, +this list of conditions and the following disclaimer in the documentation +and/or other materials provided with the distribution. + +3. The names "Center for Intelligent Information Retrieval" and +"University of Massachusetts" must not be used to endorse or promote products +derived from this software without prior written permission. To obtain +permission, contact info@ciir.cs.umass.edu. + +THIS SOFTWARE IS PROVIDED BY UNIVERSITY OF MASSACHUSETTS AND OTHER CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +SUCH DAMAGE. + +The following license applies to the Morfologik project: + +Copyright (c) 2006 Dawid Weiss +Copyright (c) 2007-2011 Dawid Weiss, Marcin Miłkowski +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name of Morfologik nor the names of its contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +--- + +The dictionary comes from Morfologik project. Morfologik uses data from +Polish ispell/myspell dictionary hosted at http://www.sjp.pl/slownik/en/ and +is licenced on the terms of (inter alia) LGPL and Creative Commons +ShareAlike. The part-of-speech tags were added in Morfologik project and +are not found in the data from sjp.pl. The tagset is similar to IPI PAN +tagset. + +--- + +The following license applies to the Morfeusz project, +used by org.apache.lucene.analysis.morfologik. + +BSD-licensed dictionary of Polish (SGJP) +http://sgjp.pl/morfeusz/ + +Copyright © 2011 Zygmunt Saloni, Włodzimierz Gruszczyński, + Marcin Woliński, Robert Wołosz + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDERS “AS IS” AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR +BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE +OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN +IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/plugins/analysis-ukrainian/licenses/lucene-NOTICE.txt b/plugins/analysis-ukrainian/licenses/lucene-NOTICE.txt new file mode 100644 index 00000000000..ecf08201a5e --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-NOTICE.txt @@ -0,0 +1,191 @@ +Apache Lucene +Copyright 2014 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +Includes software from other Apache Software Foundation projects, +including, but not limited to: + - Apache Ant + - Apache Jakarta Regexp + - Apache Commons + - Apache Xerces + +ICU4J, (under analysis/icu) is licensed under an MIT styles license +and Copyright (c) 1995-2008 International Business Machines Corporation and others + +Some data files (under analysis/icu/src/data) are derived from Unicode data such +as the Unicode Character Database. See http://unicode.org/copyright.html for more +details. + +Brics Automaton (under core/src/java/org/apache/lucene/util/automaton) is +BSD-licensed, created by Anders Møller. See http://www.brics.dk/automaton/ + +The levenshtein automata tables (under core/src/java/org/apache/lucene/util/automaton) were +automatically generated with the moman/finenight FSA library, created by +Jean-Philippe Barrette-LaPierre. This library is available under an MIT license, +see http://sites.google.com/site/rrettesite/moman and +http://bitbucket.org/jpbarrette/moman/overview/ + +The class org.apache.lucene.util.WeakIdentityMap was derived from +the Apache CXF project and is Apache License 2.0. + +The Google Code Prettify is Apache License 2.0. +See http://code.google.com/p/google-code-prettify/ + +JUnit (junit-4.10) is licensed under the Common Public License v. 1.0 +See http://junit.sourceforge.net/cpl-v10.html + +This product includes code (JaspellTernarySearchTrie) from Java Spelling Checkin +g Package (jaspell): http://jaspell.sourceforge.net/ +License: The BSD License (http://www.opensource.org/licenses/bsd-license.php) + +The snowball stemmers in + analysis/common/src/java/net/sf/snowball +were developed by Martin Porter and Richard Boulton. +The snowball stopword lists in + analysis/common/src/resources/org/apache/lucene/analysis/snowball +were developed by Martin Porter and Richard Boulton. +The full snowball package is available from + http://snowball.tartarus.org/ + +The KStem stemmer in + analysis/common/src/org/apache/lucene/analysis/en +was developed by Bob Krovetz and Sergio Guzman-Lara (CIIR-UMass Amherst) +under the BSD-license. + +The Arabic,Persian,Romanian,Bulgarian, and Hindi analyzers (common) come with a default +stopword list that is BSD-licensed created by Jacques Savoy. These files reside in: +analysis/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/fa/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/ro/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/bg/stopwords.txt, +analysis/common/src/resources/org/apache/lucene/analysis/hi/stopwords.txt +See http://members.unine.ch/jacques.savoy/clef/index.html. + +The German,Spanish,Finnish,French,Hungarian,Italian,Portuguese,Russian and Swedish light stemmers +(common) are based on BSD-licensed reference implementations created by Jacques Savoy and +Ljiljana Dolamic. These files reside in: +analysis/common/src/java/org/apache/lucene/analysis/de/GermanLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/de/GermanMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/es/SpanishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchMinimalStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/it/ItalianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/ru/RussianLightStemmer.java +analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishLightStemmer.java + +The Stempel analyzer (stempel) includes BSD-licensed software developed +by the Egothor project http://egothor.sf.net/, created by Leo Galambos, Martin Kvapil, +and Edmond Nolan. + +The Polish analyzer (stempel) comes with a default +stopword list that is BSD-licensed created by the Carrot2 project. The file resides +in stempel/src/resources/org/apache/lucene/analysis/pl/stopwords.txt. +See http://project.carrot2.org/license.html. + +The SmartChineseAnalyzer source code (smartcn) was +provided by Xiaoping Gao and copyright 2009 by www.imdict.net. + +WordBreakTestUnicode_*.java (under modules/analysis/common/src/test/) +is derived from Unicode data such as the Unicode Character Database. +See http://unicode.org/copyright.html for more details. + +The Morfologik analyzer (morfologik) includes BSD-licensed software +developed by Dawid Weiss and Marcin Miłkowski (http://morfologik.blogspot.com/). + +Morfologik uses data from Polish ispell/myspell dictionary +(http://www.sjp.pl/slownik/en/) licenced on the terms of (inter alia) +LGPL and Creative Commons ShareAlike. + +Morfologic includes data from BSD-licensed dictionary of Polish (SGJP) +(http://sgjp.pl/morfeusz/) + +Servlet-api.jar and javax.servlet-*.jar are under the CDDL license, the original +source code for this can be found at http://www.eclipse.org/jetty/downloads.php + +=========================================================================== +Kuromoji Japanese Morphological Analyzer - Apache Lucene Integration +=========================================================================== + +This software includes a binary and/or source version of data from + + mecab-ipadic-2.7.0-20070801 + +which can be obtained from + + http://atilika.com/releases/mecab-ipadic/mecab-ipadic-2.7.0-20070801.tar.gz + +or + + http://jaist.dl.sourceforge.net/project/mecab/mecab-ipadic/2.7.0-20070801/mecab-ipadic-2.7.0-20070801.tar.gz + +=========================================================================== +mecab-ipadic-2.7.0-20070801 Notice +=========================================================================== + +Nara Institute of Science and Technology (NAIST), +the copyright holders, disclaims all warranties with regard to this +software, including all implied warranties of merchantability and +fitness, in no event shall NAIST be liable for +any special, indirect or consequential damages or any damages +whatsoever resulting from loss of use, data or profits, whether in an +action of contract, negligence or other tortuous action, arising out +of or in connection with the use or performance of this software. + +A large portion of the dictionary entries +originate from ICOT Free Software. The following conditions for ICOT +Free Software applies to the current dictionary as well. + +Each User may also freely distribute the Program, whether in its +original form or modified, to any third party or parties, PROVIDED +that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear +on, or be attached to, the Program, which is distributed substantially +in the same form as set out herein and that such intended +distribution, if actually made, will neither violate or otherwise +contravene any of the laws and regulations of the countries having +jurisdiction over the User or the intended distribution itself. + +NO WARRANTY + +The program was produced on an experimental basis in the course of the +research and development conducted during the project and is provided +to users as so produced on an experimental basis. Accordingly, the +program is provided without any warranty whatsoever, whether express, +implied, statutory or otherwise. The term "warranty" used herein +includes, but is not limited to, any warranty of the quality, +performance, merchantability and fitness for a particular purpose of +the program and the nonexistence of any infringement or violation of +any right of any third party. + +Each user of the program will agree and understand, and be deemed to +have agreed and understood, that there is no warranty whatsoever for +the program and, accordingly, the entire risk arising from or +otherwise connected with the program is assumed by the user. + +Therefore, neither ICOT, the copyright holder, or any other +organization that participated in or was otherwise related to the +development of the program and their respective officials, directors, +officers and other employees shall be held liable for any and all +damages, including, without limitation, general, special, incidental +and consequential damages, arising out of or otherwise in connection +with the use or inability to use the program or any product, material +or result produced or otherwise obtained by using the program, +regardless of whether they have been advised of, or otherwise had +knowledge of, the possibility of such damages at any time during the +project or thereafter. Each user will be deemed to have agreed to the +foregoing by his or her commencement of use of the program. The term +"use" as used herein includes, but is not limited to, the use, +modification, copying and distribution of the program and the +production of secondary products from the program. + +In the case where the program, whether in its original form or +modified, was distributed or delivered to or received by a user from +any person, organization or entity other than ICOT, unless it makes or +grants independently of ICOT any specific warranty to the user in +writing, such person, organization or entity, will also be exempted +from and not be held liable to the user for any such damages as noted +above as far as the program is concerned. diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 new file mode 100644 index 00000000000..873661a7e73 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 @@ -0,0 +1 @@ +9005084a9bc71a7ce11618722f9509f6c669cce2 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.0.jar.sha1 new file mode 100644 index 00000000000..88f43752dba --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/morfologik-fsa-2.1.0.jar.sha1 @@ -0,0 +1 @@ +88e5993f73c102f378c711f6e47221b7a9e22d25 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.0.jar.sha1 new file mode 100644 index 00000000000..ec449346c7b --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/morfologik-stemming-2.1.0.jar.sha1 @@ -0,0 +1 @@ +94167b64752138a246cc33cbf1a3b0bfe5274b7c \ No newline at end of file diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java new file mode 100644 index 00000000000..45bf27b954b --- /dev/null +++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/index/analysis/UkrainianAnalyzerProvider.java @@ -0,0 +1,47 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.CharArraySet; +import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.env.Environment; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.analysis.AbstractIndexAnalyzerProvider; +import org.elasticsearch.index.analysis.Analysis; + +public class UkrainianAnalyzerProvider extends AbstractIndexAnalyzerProvider { + + private final UkrainianMorfologikAnalyzer analyzer; + + public UkrainianAnalyzerProvider(IndexSettings indexSettings, Environment env, String name, Settings settings) { + super(indexSettings, name, settings); + analyzer = new UkrainianMorfologikAnalyzer(Analysis.parseStopWords(env, settings, UkrainianMorfologikAnalyzer.getDefaultStopSet()), + Analysis.parseStemExclusion(settings, CharArraySet.EMPTY_SET)); + analyzer.setVersion(version); + } + + @Override + public UkrainianMorfologikAnalyzer get() { + return this.analyzer; + } + + +} diff --git a/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java new file mode 100644 index 00000000000..ff8425e201c --- /dev/null +++ b/plugins/analysis-ukrainian/src/main/java/org/elasticsearch/plugin/analysis/ukrainian/AnalysisUkrainianPlugin.java @@ -0,0 +1,39 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.plugin.analysis.ukrainian; + +import org.apache.lucene.analysis.Analyzer; +import org.elasticsearch.index.analysis.AnalyzerProvider; +import org.elasticsearch.index.analysis.UkrainianAnalyzerProvider; +import org.elasticsearch.indices.analysis.AnalysisModule.AnalysisProvider; +import org.elasticsearch.plugins.AnalysisPlugin; +import org.elasticsearch.plugins.Plugin; + +import java.util.Map; + +import static java.util.Collections.singletonMap; + +public class AnalysisUkrainianPlugin extends Plugin implements AnalysisPlugin { + + @Override + public Map>> getAnalyzers() { + return singletonMap("ukrainian", UkrainianAnalyzerProvider::new); + } +} diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/SimpleUkrainianAnalyzerTests.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/SimpleUkrainianAnalyzerTests.java new file mode 100644 index 00000000000..6dbc37ea4ab --- /dev/null +++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/SimpleUkrainianAnalyzerTests.java @@ -0,0 +1,56 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin; +import org.elasticsearch.test.ESTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class SimpleUkrainianAnalyzerTests extends ESTestCase { + + public void testBasicUsage() throws Exception { + testAnalyzer("чергу", "черга"); + testAnalyzer("рухається", "рухатися"); + testAnalyzer("колу", "кола", "коло", "кіл"); + testAnalyzer("Ця п'єса у свою чергу рухається по колу.", "п'єса", "черга", "рухатися", "кола", "коло", "кіл"); + } + + private static void testAnalyzer(String source, String... expected_terms) throws IOException { + TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, new AnalysisUkrainianPlugin()); + Analyzer analyzer = analysis.indexAnalyzers.get("ukrainian").analyzer(); + TokenStream ts = analyzer.tokenStream("test", source); + CharTermAttribute term1 = ts.addAttribute(CharTermAttribute.class); + ts.reset(); + for (String expected : expected_terms) { + assertThat(ts.incrementToken(), equalTo(true)); + assertThat(term1.toString(), equalTo(expected)); + } + assertThat(ts.incrementToken(), equalTo(false)); + } + +} diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java new file mode 100644 index 00000000000..a45549c22bd --- /dev/null +++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianAnalysisTests.java @@ -0,0 +1,43 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.uk.UkrainianMorfologikAnalyzer; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.Index; +import org.elasticsearch.plugin.analysis.ukrainian.AnalysisUkrainianPlugin; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.MatcherAssert; + +import java.io.IOException; + +import static org.hamcrest.Matchers.instanceOf; + +public class UkrainianAnalysisTests extends ESTestCase { + + public void testDefaultsUkranianAnalysis() throws IOException { + final TestAnalysis analysis = createTestAnalysis(new Index("test", "_na_"), Settings.EMPTY, + new AnalysisUkrainianPlugin()); + + Analyzer analyzer = analysis.indexAnalyzers.get("ukrainian").analyzer(); + MatcherAssert.assertThat(analyzer, instanceOf(UkrainianMorfologikAnalyzer.class)); + } +} diff --git a/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java new file mode 100644 index 00000000000..590d3614b97 --- /dev/null +++ b/plugins/analysis-ukrainian/src/test/java/org/elasticsearch/index/analysis/UkrainianClientYamlTestSuiteIT.java @@ -0,0 +1,42 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.analysis; + +import com.carrotsearch.randomizedtesting.annotations.Name; +import com.carrotsearch.randomizedtesting.annotations.ParametersFactory; + +import org.elasticsearch.test.rest.yaml.ClientYamlTestCandidate; +import org.elasticsearch.test.rest.yaml.ESClientYamlSuiteTestCase; +import org.elasticsearch.test.rest.yaml.parser.ClientYamlTestParseException; + +import java.io.IOException; + +public class UkrainianClientYamlTestSuiteIT extends ESClientYamlSuiteTestCase { + + public UkrainianClientYamlTestSuiteIT(@Name("yaml") ClientYamlTestCandidate testCandidate) { + super(testCandidate); + } + + @ParametersFactory + public static Iterable parameters() throws IOException, ClientYamlTestParseException { + return ESClientYamlSuiteTestCase.createParameters(0, 1); + } +} + diff --git a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/10_basic.yaml b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/10_basic.yaml new file mode 100644 index 00000000000..48d513c140c --- /dev/null +++ b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/10_basic.yaml @@ -0,0 +1,18 @@ +# Integration tests for Ukrainian analyzer +--- +"Analyzer": + - do: + indices.analyze: + body: + text: колу + analyzer: ukrainian + - length: { tokens: 3 } + - match: { tokens.0.token: кола } + - match: { tokens.0.start_offset: 0 } + - match: { tokens.0.end_offset: 4 } + - match: { tokens.1.token: коло } + - match: { tokens.1.start_offset: 0 } + - match: { tokens.1.end_offset: 4 } + - match: { tokens.2.token: кіл } + - match: { tokens.2.start_offset: 0 } + - match: { tokens.2.end_offset: 4 } diff --git a/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yaml b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yaml new file mode 100644 index 00000000000..34d8fd2fde7 --- /dev/null +++ b/plugins/analysis-ukrainian/src/test/resources/rest-api-spec/test/analysis_ukrainian/20_search.yaml @@ -0,0 +1,32 @@ +# Integration tests for Stempel analysis component +# +--- +"Index Stempel content": + - do: + indices.create: + index: test + body: + mappings: + type: + properties: + text: + type: text + analyzer: ukrainian + + - do: + index: + index: test + type: type + id: 1 + body: { "text": "Ця п'єса у свою чергу рухається по колу." } + - do: + indices.refresh: {} + + - do: + search: + index: test + body: + query: + match: + text: кола + - match: { hits.total: 1 } diff --git a/settings.gradle b/settings.gradle index 81513fd372a..4c662ac448f 100644 --- a/settings.gradle +++ b/settings.gradle @@ -36,6 +36,7 @@ List projects = [ 'plugins:analysis-phonetic', 'plugins:analysis-smartcn', 'plugins:analysis-stempel', + 'plugins:analysis-ukrainian', 'plugins:discovery-azure-classic', 'plugins:discovery-ec2', 'plugins:discovery-file', From 1f5adaa82472d21ba2811a9d62e46eaf168016fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 28 Oct 2016 17:19:33 +0200 Subject: [PATCH 02/13] Docs: Adding Ukrainian analyzer --- docs/plugins/analysis-ukrainian.asciidoc | 42 +++++++++++++++++++ docs/plugins/analysis.asciidoc | 5 +++ docs/reference/cat/plugins.asciidoc | 1 + ...lucene-analyzers-morfologik-6.2.0.jar.sha1 | 1 - ...morfologik-6.3.0-snapshot-ed102d6.jar.sha1 | 1 + 5 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 docs/plugins/analysis-ukrainian.asciidoc delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.3.0-snapshot-ed102d6.jar.sha1 diff --git a/docs/plugins/analysis-ukrainian.asciidoc b/docs/plugins/analysis-ukrainian.asciidoc new file mode 100644 index 00000000000..78f8232f1c1 --- /dev/null +++ b/docs/plugins/analysis-ukrainian.asciidoc @@ -0,0 +1,42 @@ +[[analysis-ukrainian]] +=== Ukrainian Analysis Plugin + +The Ukrainian Analysis plugin integrates Lucene's UkrainianMorfologikAnalyzer into elasticsearch. + +It provides stemming for Ukrainian using the http://github.com/morfologik/morfologik-stemming[Morfologik project]. + +[[analysis-ukrainian-install]] +[float] +==== Installation + +This plugin can be installed using the plugin manager: + +[source,sh] +---------------------------------------------------------------- +sudo bin/elasticsearch-plugin install analysis-ukrainian +---------------------------------------------------------------- + +The plugin must be installed on every node in the cluster, and each node must +be restarted after installation. + +This plugin can be downloaded for <> from +{plugin_url}/analysis-ukrainian/analysis-ukrainian-{version}.zip. + +[[analysis-ukrainian-remove]] +[float] +==== Removal + +The plugin can be removed with the following command: + +[source,sh] +---------------------------------------------------------------- +sudo bin/elasticsearch-plugin remove analysis-ukrainian +---------------------------------------------------------------- + +The node must be stopped before removing the plugin. + +[[analysis-ukrainian-analyzer]] +[float] +==== `ukrainian` analyzer + +The plugin provides the `ukrainian` analyzer. diff --git a/docs/plugins/analysis.asciidoc b/docs/plugins/analysis.asciidoc index 884dc2aebae..df68be57ab9 100644 --- a/docs/plugins/analysis.asciidoc +++ b/docs/plugins/analysis.asciidoc @@ -36,6 +36,10 @@ segmented into words. Provides high quality stemming for Polish. +<>:: + +Provides stemming for Ukrainian. + [float] ==== Community contributed analysis plugins @@ -62,5 +66,6 @@ include::analysis-smartcn.asciidoc[] include::analysis-stempel.asciidoc[] +include::analysis-ukrainian.asciidoc[] diff --git a/docs/reference/cat/plugins.asciidoc b/docs/reference/cat/plugins.asciidoc index b2a193dfa8c..b4aa02a7af7 100644 --- a/docs/reference/cat/plugins.asciidoc +++ b/docs/reference/cat/plugins.asciidoc @@ -19,6 +19,7 @@ U7321H6 analysis-kuromoji {version} The Japanese (kuromoji) Analysis plugi U7321H6 analysis-phonetic {version} The Phonetic Analysis plugin integrates phonetic token filter analysis with elasticsearch. U7321H6 analysis-smartcn {version} Smart Chinese Analysis plugin integrates Lucene Smart Chinese analysis module into elasticsearch. U7321H6 analysis-stempel {version} The Stempel (Polish) Analysis plugin integrates Lucene stempel (polish) analysis module into elasticsearch. +U7321H6 analysis-ukrainian {version} The Ukrainian Analysis plugin integrates the Lucene UkrainianMorfologikAnalyzer into elasticsearch. U7321H6 discovery-azure-classic {version} The Azure Classic Discovery plugin allows to use Azure Classic API for the unicast discovery mechanism U7321H6 discovery-ec2 {version} The EC2 discovery plugin allows to use AWS API for the unicast discovery mechanism. U7321H6 discovery-file {version} Discovery file plugin enables unicast discovery from hosts stored in a file. diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 deleted file mode 100644 index 873661a7e73..00000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.2.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -9005084a9bc71a7ce11618722f9509f6c669cce2 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.3.0-snapshot-ed102d6.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.3.0-snapshot-ed102d6.jar.sha1 new file mode 100644 index 00000000000..af4d754834c --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-6.3.0-snapshot-ed102d6.jar.sha1 @@ -0,0 +1 @@ +a08e5ddd375ea17a878ba077cc33ce59e68758c2 \ No newline at end of file From c10a6ddec112737fa3b4fa265480eef8e3eaa1a0 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Mon, 31 Oct 2016 20:04:33 +0100 Subject: [PATCH 03/13] IndexService#maybeRefresh should catch `IndexShardClosedException` (#21205) We throw this exception in some cases that the shard is closed, so we have to be consistent here. Otherwise we get logs like: ``` 1> [2016-10-30T21:06:22,529][WARN ][o.e.i.IndexService ] [node_s_0] [test] failed to run task refresh - suppressing re-occurring exceptions unless the exception changes 1> org.elasticsearch.index.shard.IndexShardClosedException: CurrentState[CLOSED] operation only allowed when not closed 1> at org.elasticsearch.index.shard.IndexShard.verifyNotClosed(IndexShard.java:1147) ~[main/:?] 1> at org.elasticsearch.index.shard.IndexShard.verifyNotClosed(IndexShard.java:1141) ~[main/:?] ``` --- core/src/main/java/org/elasticsearch/index/IndexService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/core/src/main/java/org/elasticsearch/index/IndexService.java b/core/src/main/java/org/elasticsearch/index/IndexService.java index ffcc7648293..0114d68b4e2 100644 --- a/core/src/main/java/org/elasticsearch/index/IndexService.java +++ b/core/src/main/java/org/elasticsearch/index/IndexService.java @@ -52,6 +52,7 @@ import org.elasticsearch.index.query.QueryShardContext; import org.elasticsearch.index.shard.IndexEventListener; import org.elasticsearch.index.shard.IndexSearcherWrapper; import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.IndexShardClosedException; import org.elasticsearch.index.shard.IndexingOperationListener; import org.elasticsearch.index.shard.SearchOperationListener; import org.elasticsearch.index.shard.ShadowIndexShard; @@ -692,7 +693,7 @@ public class IndexService extends AbstractIndexComponent implements IndicesClust if (shard.isRefreshNeeded()) { shard.refresh("schedule"); } - } catch (EngineClosedException | AlreadyClosedException ex) { + } catch (IndexShardClosedException | AlreadyClosedException ex) { // fine - continue; } continue; From 1d8d8ea55f7c4845c98e083a6572d256cd2e2e96 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 31 Oct 2016 16:34:21 -0400 Subject: [PATCH 04/13] Clarify production mode for bootstrap checks This commit clarifies that production mode for the bootstrap checks is only tripped if transport is bound to an external interface. Relates #21220 --- .../reference/setup/bootstrap-checks.asciidoc | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index 8c1bab474c8..cb9b8f106c7 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -23,14 +23,20 @@ documented individually. [float] === Development vs. production mode -By default, Elasticsearch binds and publishes to `localhost`. This is +By default, Elasticsearch binds to `localhost` for <> +and <> communication. This is fine for downloading and playing with Elasticsearch, and everyday -development but it's useless for production systems. For a production -installation to be reachable, it must either bind or publish to an -external interface. Thus, we consider Elasticsearch to be in development -mode if it does not bind nor publish to an external interface (the -default), and is otherwise in production mode if it does bind or publish -to an external interface. +development but it's useless for production systems. To form a cluster, +Elasticsearch instances must be reachable via transport communication so +they must bind transport to an external interface. Thus, we consider an +Elaticsearch instance to be in development mode if it does not bind +transport to an external interface (the default), and is otherwise in +production mode if it does bind transport to an external interface. Note +that HTTP can be configured independently of transport via +<> and +<>; this can be useful for +configuring a single instance to be reachable via HTTP for testing +purposes without triggering production mode. === Heap size check From 344a8028f80baf85c7e118b07230c9ba3e334e03 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 31 Oct 2016 16:38:33 -0400 Subject: [PATCH 05/13] Add analysis-ukrainian plugin to packaging tests This commit adds the analysis-ukrainian plugin to the packaging tests. Relates #21219 --- .../scripts/module_and_plugin_test_cases.bash | 8 ++++++++ .../resources/packaging/scripts/plugins.bash | 20 +++++++++++++------ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash index d5dc095e70f..fbc0f4a887a 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/module_and_plugin_test_cases.bash @@ -205,6 +205,10 @@ fi install_and_check_plugin analysis stempel } +@test "[$GROUP] install ukrainian plugin" { + install_and_check_plugin analysis ukrainian morfologik-fsa-*.jar morfologik-stemming-*.jar +} + @test "[$GROUP] install gce plugin" { install_and_check_plugin discovery gce google-api-client-*.jar } @@ -341,6 +345,10 @@ fi remove_plugin analysis-stempel } +@test "[$GROUP] remove ukrainian plugin" { + remove_plugin analysis-ukrainian +} + @test "[$GROUP] remove gce plugin" { remove_plugin discovery-gce } diff --git a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash index 02cae2aeecb..55e7fdfc484 100644 --- a/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash +++ b/qa/vagrant/src/test/resources/packaging/scripts/plugins.bash @@ -129,20 +129,28 @@ install_and_check_plugin() { shift if [ "$prefix" == "-" ]; then - local fullName="$name" + local full_name="$name" else - local fullName="$prefix-$name" + local full_name="$prefix-$name" fi - install_jvm_plugin $fullName "$(readlink -m $fullName-*.zip)" + install_jvm_plugin $full_name "$(readlink -m $full_name-*.zip)" - assert_module_or_plugin_directory "$ESPLUGINS/$fullName" + assert_module_or_plugin_directory "$ESPLUGINS/$full_name" + # analysis plugins have a corresponding analyzers jar if [ $prefix == 'analysis' ]; then - assert_module_or_plugin_file "$ESPLUGINS/$fullName/lucene-analyzers-$name-*.jar" + local analyzer_jar_suffix=$name + # the name of the analyzer jar for the ukrainian plugin does + # not match the name of the plugin, so we have to make an + # exception + if [ $name == 'ukrainian' ]; then + analyzer_jar_suffix='morfologik' + fi + assert_module_or_plugin_file "$ESPLUGINS/$full_name/lucene-analyzers-$analyzer_jar_suffix-*.jar" fi for file in "$@"; do - assert_module_or_plugin_file "$ESPLUGINS/$fullName/$file" + assert_module_or_plugin_file "$ESPLUGINS/$full_name/$file" done } From 185dff73461ae384f44fda5c7883fa6ba5e8c466 Mon Sep 17 00:00:00 2001 From: Jack Conradson Date: Mon, 31 Oct 2016 13:48:51 -0700 Subject: [PATCH 06/13] Cleanup ScriptType (#21179) Refactored ScriptType to clean up some of the variable and method names. Added more documentation. Deprecated the 'in' ParseField in favor of 'stored' to match the indexed scripts being replaced by stored scripts. --- .../java/org/elasticsearch/script/Script.java | 2 +- .../org/elasticsearch/script/ScriptModes.java | 2 +- .../elasticsearch/script/ScriptSettings.java | 4 +- .../org/elasticsearch/script/ScriptType.java | 128 ++++++++++++------ .../script/NativeScriptTests.java | 2 +- .../script/ScriptModesTests.java | 2 +- .../script/ScriptServiceTests.java | 4 +- docs/plugins/lang-javascript.asciidoc | 2 +- docs/plugins/lang-python.asciidoc | 2 +- .../modules/scripting/using.asciidoc | 8 +- .../query-dsl/template-query.asciidoc | 2 +- .../mustache/SearchTemplateRequest.java | 2 +- .../test/lang_mustache/40_template_query.yaml | 4 +- 13 files changed, 107 insertions(+), 57 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/script/Script.java b/core/src/main/java/org/elasticsearch/script/Script.java index cfdbf966b05..edcf6666817 100644 --- a/core/src/main/java/org/elasticsearch/script/Script.java +++ b/core/src/main/java/org/elasticsearch/script/Script.java @@ -109,7 +109,7 @@ public final class Script implements ToXContent, Writeable { boolean hasType = type != null; out.writeBoolean(hasType); if (hasType) { - ScriptType.writeTo(type, out); + type.writeTo(out); } out.writeOptionalString(lang); out.writeMap(params); diff --git a/core/src/main/java/org/elasticsearch/script/ScriptModes.java b/core/src/main/java/org/elasticsearch/script/ScriptModes.java index 4f9651b290a..15393948d66 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptModes.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptModes.java @@ -72,7 +72,7 @@ public class ScriptModes { } static String sourceKey(ScriptType scriptType) { - return SCRIPT_SETTINGS_PREFIX + "." + scriptType.getScriptType(); + return SCRIPT_SETTINGS_PREFIX + "." + scriptType.getName(); } static String getGlobalKey(String lang, ScriptType scriptType) { diff --git a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java index 07e0deb5b71..27a6ad04a70 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptSettings.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptSettings.java @@ -50,7 +50,7 @@ public class ScriptSettings { for (ScriptType scriptType : ScriptType.values()) { scriptTypeSettingMap.put(scriptType, Setting.boolSetting( ScriptModes.sourceKey(scriptType), - scriptType.getDefaultScriptEnabled(), + scriptType.isDefaultEnabled(), Property.NodeScope)); } SCRIPT_TYPE_SETTING_MAP = Collections.unmodifiableMap(scriptTypeSettingMap); @@ -102,7 +102,7 @@ public class ScriptSettings { boolean defaultLangAndType = defaultNonFileScriptMode; // Files are treated differently because they are never default-deny if (ScriptType.FILE == scriptType) { - defaultLangAndType = ScriptType.FILE.getDefaultScriptEnabled(); + defaultLangAndType = ScriptType.FILE.isDefaultEnabled(); } final boolean defaultIfNothingSet = defaultLangAndType; diff --git a/core/src/main/java/org/elasticsearch/script/ScriptType.java b/core/src/main/java/org/elasticsearch/script/ScriptType.java index 77865daa372..01592b57aad 100644 --- a/core/src/main/java/org/elasticsearch/script/ScriptType.java +++ b/core/src/main/java/org/elasticsearch/script/ScriptType.java @@ -22,68 +22,118 @@ package org.elasticsearch.script; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; import java.io.IOException; -import java.util.Locale; /** - * The type of a script, more specifically where it gets loaded from: - * - provided dynamically at request time - * - loaded from an index - * - loaded from file + * ScriptType represents the way a script is stored and retrieved from the {@link ScriptService}. + * It's also used to by {@link ScriptSettings} and {@link ScriptModes} to determine whether or not + * a {@link Script} is allowed to be executed based on both default and user-defined settings. */ -public enum ScriptType { +public enum ScriptType implements Writeable { - INLINE(0, "inline", "inline", false), - STORED(1, "id", "stored", false), - FILE(2, "file", "file", true); + /** + * INLINE scripts are specified in numerous queries and compiled on-the-fly. + * They will be cached based on the lang and code of the script. + * They are turned off by default because most languages are insecure + * (Groovy and others), but can be overriden by the specific {@link ScriptEngineService} + * if the language is naturally secure (Painless, Mustache, and Expressions). + */ + INLINE ( 0 , new ParseField("inline") , false ), - private final int val; - private final ParseField parseField; - private final String scriptType; - private final boolean defaultScriptEnabled; + /** + * STORED scripts are saved as part of the {@link org.elasticsearch.cluster.ClusterState} + * based on user requests. They will be cached when they are first used in a query. + * They are turned off by default because most languages are insecure + * (Groovy and others), but can be overriden by the specific {@link ScriptEngineService} + * if the language is naturally secure (Painless, Mustache, and Expressions). + */ + STORED ( 1 , new ParseField("stored", "id") , false ), + /** + * FILE scripts are loaded from disk either on start-up or on-the-fly depending on + * user-defined settings. They will be compiled and cached as soon as they are loaded + * from disk. They are turned on by default as they should always be safe to execute. + */ + FILE ( 2 , new ParseField("file") , true ); + + /** + * Reads an int from the input stream and converts it to a {@link ScriptType}. + * @return The ScriptType read from the stream. Throws an {@link IllegalStateException} + * if no ScriptType is found based on the id. + */ public static ScriptType readFrom(StreamInput in) throws IOException { - int scriptTypeVal = in.readVInt(); - for (ScriptType type : values()) { - if (type.val == scriptTypeVal) { - return type; - } - } - throw new IllegalArgumentException("Unexpected value read for ScriptType got [" + scriptTypeVal + "] expected one of [" - + INLINE.val + "," + FILE.val + "," + STORED.val + "]"); - } + int id = in.readVInt(); - public static void writeTo(ScriptType scriptType, StreamOutput out) throws IOException{ - if (scriptType != null) { - out.writeVInt(scriptType.val); + if (FILE.id == id) { + return FILE; + } else if (STORED.id == id) { + return STORED; + } else if (INLINE.id == id) { + return INLINE; } else { - out.writeVInt(INLINE.val); //Default to inline + throw new IllegalStateException("Error reading ScriptType id [" + id + "] from stream, expected one of [" + + FILE.id + " [" + FILE.parseField.getPreferredName() + "], " + + STORED.id + " [" + STORED.parseField.getPreferredName() + "], " + + INLINE.id + " [" + INLINE.parseField.getPreferredName() + "]]"); } } - ScriptType(int val, String name, String scriptType, boolean defaultScriptEnabled) { - this.val = val; - this.parseField = new ParseField(name); - this.scriptType = scriptType; - this.defaultScriptEnabled = defaultScriptEnabled; + private final int id; + private final ParseField parseField; + private final boolean defaultEnabled; + + /** + * Standard constructor. + * @param id A unique identifier for a type that can be read/written to a stream. + * @param parseField Specifies the name used to parse input from queries. + * @param defaultEnabled Whether or not a {@link ScriptType} can be run by default. + */ + ScriptType(int id, ParseField parseField, boolean defaultEnabled) { + this.id = id; + this.parseField = parseField; + this.defaultEnabled = defaultEnabled; } + public void writeTo(StreamOutput out) throws IOException { + out.writeVInt(id); + } + + /** + * @return The unique id for this {@link ScriptType}. + */ + public int getId() { + return id; + } + + /** + * @return The unique name for this {@link ScriptType} based on the {@link ParseField}. + */ + public String getName() { + return parseField.getPreferredName(); + } + + /** + * @return Specifies the name used to parse input from queries. + */ public ParseField getParseField() { return parseField; } - public boolean getDefaultScriptEnabled() { - return defaultScriptEnabled; - } - - public String getScriptType() { - return scriptType; + /** + * @return Whether or not a {@link ScriptType} can be run by default. Note + * this can be potentially overriden by any {@link ScriptEngineService}. + */ + public boolean isDefaultEnabled() { + return defaultEnabled; } + /** + * @return The same as calling {@link #getName()}. + */ @Override public String toString() { - return name().toLowerCase(Locale.ROOT); + return getName(); } - } diff --git a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java index eeafbde8850..efff2b0834e 100644 --- a/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java +++ b/core/src/test/java/org/elasticsearch/script/NativeScriptTests.java @@ -62,7 +62,7 @@ public class NativeScriptTests extends ESTestCase { Settings.Builder builder = Settings.builder(); if (randomBoolean()) { ScriptType scriptType = randomFrom(ScriptType.values()); - builder.put("script" + "." + scriptType.getScriptType(), randomBoolean()); + builder.put("script" + "." + scriptType.getName(), randomBoolean()); } else { ScriptContext scriptContext = randomFrom(ScriptContext.Standard.values()); builder.put("script" + "." + scriptContext.getKey(), randomBoolean()); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java index a56d056cd6b..f6a02ae9206 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptModesTests.java @@ -125,7 +125,7 @@ public class ScriptModesTests extends ESTestCase { ScriptType[] randomScriptTypes = randomScriptTypesSet.toArray(new ScriptType[randomScriptTypesSet.size()]); Settings.Builder builder = Settings.builder(); for (int i = 0; i < randomInt; i++) { - builder.put("script" + "." + randomScriptTypes[i].getScriptType(), randomScriptModes[i]); + builder.put("script" + "." + randomScriptTypes[i].getName(), randomScriptModes[i]); } this.scriptModes = new ScriptModes(scriptSettings, builder.build()); diff --git a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java index 8ed5f4c957d..2b14ba3f4d0 100644 --- a/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java +++ b/core/src/test/java/org/elasticsearch/script/ScriptServiceTests.java @@ -266,9 +266,9 @@ public class ScriptServiceTests extends ESTestCase { Settings.Builder builder = Settings.builder(); for (Map.Entry entry : scriptSourceSettings.entrySet()) { if (entry.getValue()) { - builder.put("script" + "." + entry.getKey().getScriptType(), "true"); + builder.put("script" + "." + entry.getKey().getName(), "true"); } else { - builder.put("script" + "." + entry.getKey().getScriptType(), "false"); + builder.put("script" + "." + entry.getKey().getName(), "false"); } } for (Map.Entry entry : scriptContextSettings.entrySet()) { diff --git a/docs/plugins/lang-javascript.asciidoc b/docs/plugins/lang-javascript.asciidoc index 64be9dbb033..001b7f22231 100644 --- a/docs/plugins/lang-javascript.asciidoc +++ b/docs/plugins/lang-javascript.asciidoc @@ -120,7 +120,7 @@ GET test/_search "function_score": { "script_score": { "script": { - "id": "my_script", <2> + "stored": "my_script", <2> "lang": "javascript", "params": { "factor": 2 diff --git a/docs/plugins/lang-python.asciidoc b/docs/plugins/lang-python.asciidoc index 0730f8b54b7..0e328d79f60 100644 --- a/docs/plugins/lang-python.asciidoc +++ b/docs/plugins/lang-python.asciidoc @@ -119,7 +119,7 @@ GET test/_search "function_score": { "script_score": { "script": { - "id": "my_script", <2> + "stored": "my_script", <2> "lang": "python", "params": { "factor": 2 diff --git a/docs/reference/modules/scripting/using.asciidoc b/docs/reference/modules/scripting/using.asciidoc index c3af5861879..b09a54e5c46 100644 --- a/docs/reference/modules/scripting/using.asciidoc +++ b/docs/reference/modules/scripting/using.asciidoc @@ -8,12 +8,12 @@ the same pattern: ------------------------------------- "script": { "lang": "...", <1> - "inline" | "id" | "file": "...", <2> + "inline" | "stored" | "file": "...", <2> "params": { ... } <3> } ------------------------------------- <1> The language the script is written in, which defaults to `painless`. -<2> The script itself which may be specified as `inline`, `id`, or `file`. +<2> The script itself which may be specified as `inline`, `stored`, or `file`. <3> Any named parameters that should be passed into the script. For example, the following script is used in a search request to return a @@ -211,7 +211,7 @@ GET _scripts/groovy/calculate-score // CONSOLE // TEST[continued] -Stored scripts can be used by specifying the `lang` and `id` parameters as follows: +Stored scripts can be used by specifying the `lang` and `stored` parameters as follows: [source,js] -------------------------------------------------- @@ -221,7 +221,7 @@ GET _search "script": { "script": { "lang": "groovy", - "id": "calculate-score", + "stored": "calculate-score", "params": { "my_modifier": 2 } diff --git a/docs/reference/query-dsl/template-query.asciidoc b/docs/reference/query-dsl/template-query.asciidoc index b4b00e5babd..2d3b5724d49 100644 --- a/docs/reference/query-dsl/template-query.asciidoc +++ b/docs/reference/query-dsl/template-query.asciidoc @@ -108,7 +108,7 @@ GET /_search { "query": { "template": { - "id": "my_template", <1> + "stored": "my_template", <1> "params" : { "query_string" : "all about search" } diff --git a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java index 249b0080775..d7ac37f8313 100644 --- a/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java +++ b/modules/lang-mustache/src/main/java/org/elasticsearch/script/mustache/SearchTemplateRequest.java @@ -157,7 +157,7 @@ public class SearchTemplateRequest extends ActionRequest out.writeBoolean(simulate); out.writeBoolean(explain); out.writeBoolean(profile); - ScriptType.writeTo(scriptType, out); + scriptType.writeTo(out); out.writeOptionalString(script); boolean hasParams = scriptParams != null; out.writeBoolean(hasParams); diff --git a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml index 2360dfc37f0..cfa97b8bc9f 100644 --- a/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml +++ b/modules/lang-mustache/src/test/resources/rest-api-spec/test/lang_mustache/40_template_query.yaml @@ -44,7 +44,7 @@ warnings: - '[template] query is deprecated, use search template api instead' search: - body: { "query": { "template": { "id": "1", "params": { "my_value": "value1" } } } } + body: { "query": { "template": { "stored": "1", "params": { "my_value": "value1" } } } } - match: { hits.total: 1 } @@ -52,7 +52,7 @@ warnings: - '[template] query is deprecated, use search template api instead' search: - body: { "query": { "template": { "id": "/mustache/1", "params": { "my_value": "value1" } } } } + body: { "query": { "template": { "stored": "/mustache/1", "params": { "my_value": "value1" } } } } - match: { hits.total: 1 } From 38663351dcf76955e8ef6f4bc959d777825ec425 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 31 Oct 2016 17:23:21 -0400 Subject: [PATCH 07/13] Fix logger names for Netty Previously Elasticsearch would only use the package name for logging levels, truncating the package prefix and the class name. This meant that logger names for Netty were just prefixed by netty3 and netty. We changed this for Elasticsearch so that it's the fully-qualified class name now, but never corrected this for Netty. This commit fixes the logger names for the Netty modules so that their levels are controlled by the fully-qualified class name. Relates #21223 --- .../java/org/elasticsearch/transport/netty3/Netty3Utils.java | 1 - .../java/org/elasticsearch/transport/netty4/Netty4Utils.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java index aea5b31f457..f32bd5dc19b 100644 --- a/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java +++ b/modules/transport-netty3/src/main/java/org/elasticsearch/transport/netty3/Netty3Utils.java @@ -99,7 +99,6 @@ public class Netty3Utils { InternalLoggerFactory.setDefaultFactory(new InternalLoggerFactory() { @Override public InternalLogger newInstance(String name) { - name = name.replace("org.jboss.netty.", "netty3.").replace("org.jboss.netty.", "netty3."); return new Netty3InternalESLogger(Loggers.getLogger(name)); } }); diff --git a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java index 53cf1b329aa..877d50e1674 100644 --- a/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java +++ b/modules/transport-netty4/src/main/java/org/elasticsearch/transport/netty4/Netty4Utils.java @@ -41,7 +41,7 @@ public class Netty4Utils { @Override public InternalLogger newInstance(final String name) { - return new Netty4InternalESLogger(name.replace("io.netty.", "netty.")); + return new Netty4InternalESLogger(name); } }); From 5fab17c7bc03dd0d336c19a23bc7655aaa12fdc2 Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 31 Oct 2016 17:33:15 -0400 Subject: [PATCH 08/13] Fix markup for bootstrap checks docs This commit adds a missing backtick to the bootstrap check docs. --- docs/reference/setup/bootstrap-checks.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index cb9b8f106c7..adc58840e6d 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -33,7 +33,7 @@ Elaticsearch instance to be in development mode if it does not bind transport to an external interface (the default), and is otherwise in production mode if it does bind transport to an external interface. Note that HTTP can be configured independently of transport via -<> and +<> and <>; this can be useful for configuring a single instance to be reachable via HTTP for testing purposes without triggering production mode. From f81d28d0a05a013c3a1e0dda6c72773b4310fa2d Mon Sep 17 00:00:00 2001 From: Jason Tedor Date: Mon, 31 Oct 2016 20:48:55 -0400 Subject: [PATCH 09/13] Fix transport host setting This commit fixes a reference to the transport.host setting which was mistakenly identified as transport.tcp.host. --- docs/reference/setup/bootstrap-checks.asciidoc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/reference/setup/bootstrap-checks.asciidoc b/docs/reference/setup/bootstrap-checks.asciidoc index adc58840e6d..9c2276bc213 100644 --- a/docs/reference/setup/bootstrap-checks.asciidoc +++ b/docs/reference/setup/bootstrap-checks.asciidoc @@ -33,10 +33,9 @@ Elaticsearch instance to be in development mode if it does not bind transport to an external interface (the default), and is otherwise in production mode if it does bind transport to an external interface. Note that HTTP can be configured independently of transport via -<> and -<>; this can be useful for -configuring a single instance to be reachable via HTTP for testing -purposes without triggering production mode. +<> and <>; +this can be useful for configuring a single instance to be reachable via +HTTP for testing purposes without triggering production mode. === Heap size check From ef192ff2cffbf8dee6b3d2c3a12122ea75561800 Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 1 Nov 2016 12:54:20 +0100 Subject: [PATCH 10/13] ESIntegTestCase.jav: use ClusterState.prettyPrint for pending ClusterState assertions --- .../java/org/elasticsearch/test/ESIntegTestCase.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java index bde3ebaa85c..1e98754a798 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESIntegTestCase.java @@ -164,6 +164,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.function.BooleanSupplier; import java.util.function.Function; +import java.util.stream.Collectors; import static org.elasticsearch.client.Requests.syncedFlushRequest; import static org.elasticsearch.cluster.metadata.IndexMetaData.SETTING_NUMBER_OF_REPLICAS; @@ -535,13 +536,10 @@ public abstract class ESIntegTestCase extends ESTestCase { for (Discovery discovery : internalCluster().getInstances(Discovery.class)) { if (discovery instanceof ZenDiscovery) { final ZenDiscovery zenDiscovery = (ZenDiscovery) discovery; - assertBusy(new Runnable() { - @Override - public void run() { - assertThat("still having pending states: " + Strings.arrayToDelimitedString(zenDiscovery.pendingClusterStates(), "\n"), - zenDiscovery.pendingClusterStates(), emptyArray()); - } - }); + assertBusy(() -> assertThat(zenDiscovery.localNode().getName() + " still having pending states: \n" + + Arrays.stream(zenDiscovery.pendingClusterStates()).map(ClusterState::prettyPrint) + .collect(Collectors.joining("\n")), + zenDiscovery.pendingClusterStates(), emptyArray())); } } } From 6e7e89159bff9dc60b619c0ee752dd88c6d4446a Mon Sep 17 00:00:00 2001 From: Jay Modi Date: Tue, 1 Nov 2016 09:02:05 -0400 Subject: [PATCH 11/13] ensure the XContentBuilder is always closed in RestBuilderListener There may be cases where the XContentBuilder is not used and therefore it never gets closed, which can cause a leak of bytes. This change moves the creation of the builder into a try with resources block and adds an assertion to verify that we always consume the bytes in our code; the try-with resources provides protections against memory leaks caused by plugins, which do not test this. --- .../common/xcontent/XContentGenerator.java | 5 ++ .../xcontent/json/JsonXContentGenerator.java | 4 + .../rest/action/RestBuilderListener.java | 15 +++- .../rest/action/RestBuilderListenerTests.java | 90 +++++++++++++++++++ 4 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 core/src/test/java/org/elasticsearch/rest/action/RestBuilderListenerTests.java diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java index 8d1b8efef51..478f3a8a08f 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/XContentGenerator.java @@ -94,4 +94,9 @@ public interface XContentGenerator extends Closeable, Flushable { void copyCurrentStructure(XContentParser parser) throws IOException; + /** + * Returns {@code true} if this XContentGenerator has been closed. A closed generator can not do any more output. + */ + boolean isClosed(); + } diff --git a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java index 74e1cb5e58f..763fac4c6a3 100644 --- a/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java +++ b/core/src/main/java/org/elasticsearch/common/xcontent/json/JsonXContentGenerator.java @@ -419,4 +419,8 @@ public class JsonXContentGenerator implements XContentGenerator { generator.close(); } + @Override + public boolean isClosed() { + return generator.isClosed(); + } } diff --git a/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java b/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java index cc93e72d80d..c460331afaa 100644 --- a/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java +++ b/core/src/main/java/org/elasticsearch/rest/action/RestBuilderListener.java @@ -34,11 +34,22 @@ public abstract class RestBuilderListener extends RestResponseListener @Override public final RestResponse buildResponse(Response response) throws Exception { - return buildResponse(response, channel.newBuilder()); + try (XContentBuilder builder = channel.newBuilder()) { + final RestResponse restResponse = buildResponse(response, builder); + assert assertBuilderClosed(builder); + return restResponse; + } } /** - * Builds a response to send back over the channel. + * Builds a response to send back over the channel. Implementors should ensure that they close the provided {@link XContentBuilder} + * using the {@link XContentBuilder#close()} method. */ public abstract RestResponse buildResponse(Response response, XContentBuilder builder) throws Exception; + + // pkg private method that we can override for testing + boolean assertBuilderClosed(XContentBuilder xContentBuilder) { + assert xContentBuilder.generator().isClosed() : "callers should ensure the XContentBuilder is closed themselves"; + return true; + } } diff --git a/core/src/test/java/org/elasticsearch/rest/action/RestBuilderListenerTests.java b/core/src/test/java/org/elasticsearch/rest/action/RestBuilderListenerTests.java new file mode 100644 index 00000000000..2bc0d0bdc81 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/rest/action/RestBuilderListenerTests.java @@ -0,0 +1,90 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.rest.action; + +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestChannel; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.transport.TransportResponse; +import org.elasticsearch.transport.TransportResponse.Empty; + +import java.util.concurrent.atomic.AtomicReference; + +public class RestBuilderListenerTests extends ESTestCase { + + public void testXContentBuilderClosedInBuildResponse() throws Exception { + AtomicReference builderAtomicReference = new AtomicReference<>(); + RestBuilderListener builderListener = + new RestBuilderListener(new FakeRestChannel(new FakeRestRequest(), randomBoolean(), 1)) { + @Override + public RestResponse buildResponse(Empty empty, XContentBuilder builder) throws Exception { + builderAtomicReference.set(builder); + builder.close(); + return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY); + } + }; + + builderListener.buildResponse(Empty.INSTANCE); + assertNotNull(builderAtomicReference.get()); + assertTrue(builderAtomicReference.get().generator().isClosed()); + } + + public void testXContentBuilderNotClosedInBuildResponseAssertionsDisabled() throws Exception { + AtomicReference builderAtomicReference = new AtomicReference<>(); + RestBuilderListener builderListener = + new RestBuilderListener(new FakeRestChannel(new FakeRestRequest(), randomBoolean(), 1)) { + @Override + public RestResponse buildResponse(Empty empty, XContentBuilder builder) throws Exception { + builderAtomicReference.set(builder); + return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY); + } + + @Override + boolean assertBuilderClosed(XContentBuilder xContentBuilder) { + // don't check the actual builder being closed so we can test auto close + return true; + } + }; + + builderListener.buildResponse(Empty.INSTANCE); + assertNotNull(builderAtomicReference.get()); + assertTrue(builderAtomicReference.get().generator().isClosed()); + } + + public void testXContentBuilderNotClosedInBuildResponseAssertionsEnabled() throws Exception { + assumeTrue("tests are not being run with assertions", RestBuilderListener.class.desiredAssertionStatus()); + + RestBuilderListener builderListener = + new RestBuilderListener(new FakeRestChannel(new FakeRestRequest(), randomBoolean(), 1)) { + @Override + public RestResponse buildResponse(Empty empty, XContentBuilder builder) throws Exception { + return new BytesRestResponse(RestStatus.OK, BytesRestResponse.TEXT_CONTENT_TYPE, BytesArray.EMPTY); + } + }; + + AssertionError error = expectThrows(AssertionError.class, () -> builderListener.buildResponse(Empty.INSTANCE)); + assertEquals("callers should ensure the XContentBuilder is closed themselves", error.getMessage()); + } +} From 523f7ea71ea71d26474c76609ff7e20faddcb34b Mon Sep 17 00:00:00 2001 From: Boaz Leskes Date: Tue, 1 Nov 2016 13:55:55 +0100 Subject: [PATCH 12/13] Fix a racing condition in MockTransportService#addUnresponsiveRule where a request can be delayed even if the rule was removed. Relates to #21129 Also properly reset DiscoveryWithServiceDisruptionsIT#disableBeforeIndexDeletion --- .../DiscoveryWithServiceDisruptionsIT.java | 12 +++++++--- .../test/transport/MockTransportService.java | 22 ++++++++++++++----- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java index 5af2e1c6624..f0c8258864a 100644 --- a/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java +++ b/core/src/test/java/org/elasticsearch/discovery/DiscoveryWithServiceDisruptionsIT.java @@ -50,13 +50,13 @@ import org.elasticsearch.common.collect.Tuple; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.discovery.zen.ElectMasterService; -import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.discovery.zen.FaultDetection; import org.elasticsearch.discovery.zen.MembershipAction; +import org.elasticsearch.discovery.zen.PublishClusterStateAction; +import org.elasticsearch.discovery.zen.UnicastZenPing; +import org.elasticsearch.discovery.zen.ZenDiscovery; import org.elasticsearch.discovery.zen.ZenPing; import org.elasticsearch.discovery.zen.ZenPingService; -import org.elasticsearch.discovery.zen.UnicastZenPing; -import org.elasticsearch.discovery.zen.PublishClusterStateAction; import org.elasticsearch.env.NodeEnvironment; import org.elasticsearch.indices.store.IndicesStoreIntegrationIT; import org.elasticsearch.monitor.jvm.HotThreads; @@ -155,6 +155,12 @@ public class DiscoveryWithServiceDisruptionsIT extends ESIntegTestCase { private boolean disableBeforeIndexDeletion; + @Before + public void setUp() throws Exception { + super.setUp(); + disableBeforeIndexDeletion = false; + } + @Override public void setDisruptionScheme(ServiceDisruptionScheme scheme) { if (scheme instanceof NetworkDisruption && diff --git a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java index c63f968011e..dfa30874221 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java +++ b/test/framework/src/main/java/org/elasticsearch/test/transport/MockTransportService.java @@ -60,9 +60,9 @@ import java.util.List; import java.util.Map; import java.util.Queue; import java.util.Set; -import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.LinkedBlockingDeque; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -299,7 +299,8 @@ public final class MockTransportService extends TransportService { final long startTime = System.currentTimeMillis(); addDelegate(transportAddress, new ClearableTransport(original) { - private final Queue requestsToSendWhenCleared = new ConcurrentLinkedQueue<>(); + private final Queue requestsToSendWhenCleared = new LinkedBlockingDeque(); + private boolean cleared = false; TimeValue getDelay() { return new TimeValue(duration.millis() - (System.currentTimeMillis() - startTime)); @@ -386,15 +387,24 @@ public final class MockTransportService extends TransportService { }; // store the request to send it once the rule is cleared. - requestsToSendWhenCleared.add(runnable); - - threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable); + synchronized (this) { + if (cleared) { + runnable.run(); + } else { + requestsToSendWhenCleared.add(runnable); + threadPool.schedule(delay, ThreadPool.Names.GENERIC, runnable); + } + } } @Override public void clearRule() { - requestsToSendWhenCleared.forEach(Runnable::run); + synchronized (this) { + assert cleared == false; + cleared = true; + requestsToSendWhenCleared.forEach(Runnable::run); + } } }); } From 5af6deb5b5d9d876f8975e587d121808ac402193 Mon Sep 17 00:00:00 2001 From: LakumiNarayanan Date: Tue, 1 Nov 2016 19:43:53 +0530 Subject: [PATCH 13/13] Fix typo in keyword.asciidoc (#21237) --- docs/reference/mapping/types/keyword.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/mapping/types/keyword.asciidoc b/docs/reference/mapping/types/keyword.asciidoc index fa260bbeff6..7c09ef46e55 100644 --- a/docs/reference/mapping/types/keyword.asciidoc +++ b/docs/reference/mapping/types/keyword.asciidoc @@ -6,7 +6,7 @@ codes, zip codes or tags. They are typically used for filtering (_Find me all blog posts where ++status++ is ++published++_), for sorting, and for aggregations. Keyword -fields are ony searchable by their exact value. +fields are only searchable by their exact value. If you need to index full text content such as email bodies or product descriptions, it is likely that you should rather use a <> field.