From c5a3c3cb419507a803fb742e805149d2d96c1178 Mon Sep 17 00:00:00 2001 From: Nick Knize Date: Fri, 23 Apr 2021 09:48:41 -0500 Subject: [PATCH] Update lucene version to 8.8.2 (#557) This commit updates the codebase to the latest released version of Lucene. Signed-off-by: Nicholas Walter Knize --- buildSrc/version.properties | 2 +- .../lucene-expressions-8.7.0.jar.sha1 | 1 - .../lucene-expressions-8.8.2.jar.sha1 | 1 + .../lucene-analyzers-icu-8.7.0.jar.sha1 | 1 - .../lucene-analyzers-icu-8.8.2.jar.sha1 | 1 + .../lucene-analyzers-kuromoji-8.7.0.jar.sha1 | 1 - .../lucene-analyzers-kuromoji-8.8.2.jar.sha1 | 1 + .../lucene-analyzers-nori-8.7.0.jar.sha1 | 1 - .../lucene-analyzers-nori-8.8.2.jar.sha1 | 1 + .../lucene-analyzers-phonetic-8.7.0.jar.sha1 | 1 - .../lucene-analyzers-phonetic-8.8.2.jar.sha1 | 1 + .../lucene-analyzers-smartcn-8.7.0.jar.sha1 | 1 - .../lucene-analyzers-smartcn-8.8.2.jar.sha1 | 1 + .../lucene-analyzers-stempel-8.7.0.jar.sha1 | 1 - .../lucene-analyzers-stempel-8.8.2.jar.sha1 | 1 + ...lucene-analyzers-morfologik-8.7.0.jar.sha1 | 1 - ...lucene-analyzers-morfologik-8.8.2.jar.sha1 | 1 + .../lucene-analyzers-common-8.7.0.jar.sha1 | 1 - .../lucene-analyzers-common-8.8.2.jar.sha1 | 1 + .../lucene-backward-codecs-8.7.0.jar.sha1 | 1 - .../lucene-backward-codecs-8.8.2.jar.sha1 | 1 + server/licenses/lucene-core-8.7.0.jar.sha1 | 1 - server/licenses/lucene-core-8.8.2.jar.sha1 | 1 + .../licenses/lucene-grouping-8.7.0.jar.sha1 | 1 - .../licenses/lucene-grouping-8.8.2.jar.sha1 | 1 + .../lucene-highlighter-8.7.0.jar.sha1 | 1 - .../lucene-highlighter-8.8.2.jar.sha1 | 1 + server/licenses/lucene-join-8.7.0.jar.sha1 | 1 - server/licenses/lucene-join-8.8.2.jar.sha1 | 1 + server/licenses/lucene-memory-8.7.0.jar.sha1 | 1 - server/licenses/lucene-memory-8.8.2.jar.sha1 | 1 + server/licenses/lucene-misc-8.7.0.jar.sha1 | 1 - server/licenses/lucene-misc-8.8.2.jar.sha1 | 1 + server/licenses/lucene-queries-8.7.0.jar.sha1 | 1 - server/licenses/lucene-queries-8.8.2.jar.sha1 | 1 + .../lucene-queryparser-8.7.0.jar.sha1 | 1 - .../lucene-queryparser-8.8.2.jar.sha1 | 1 + server/licenses/lucene-sandbox-8.7.0.jar.sha1 | 1 - server/licenses/lucene-sandbox-8.8.2.jar.sha1 | 1 + .../lucene-spatial-extras-8.7.0.jar.sha1 | 1 - .../lucene-spatial-extras-8.8.2.jar.sha1 | 1 + .../licenses/lucene-spatial3d-8.7.0.jar.sha1 | 1 - .../licenses/lucene-spatial3d-8.8.2.jar.sha1 | 1 + server/licenses/lucene-suggest-8.7.0.jar.sha1 | 1 - server/licenses/lucene-suggest-8.8.2.jar.sha1 | 1 + .../src/main/java/org/opensearch/Version.java | 2 +- .../opensearch/index/codec/CodecService.java | 2 +- .../PerFieldMappingPostingFormatCodec.java | 10 ++++-- .../query/VectorGeoShapeQueryProcessor.java | 8 ++++- .../opensearch/index/codec/CodecTests.java | 33 ++++++++++++++++--- .../analysis/AnalysisFactoryTestCase.java | 1 + 51 files changed, 69 insertions(+), 33 deletions(-) delete mode 100644 modules/lang-expression/licenses/lucene-expressions-8.7.0.jar.sha1 create mode 100644 modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1 delete mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0.jar.sha1 create mode 100644 plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1 delete mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0.jar.sha1 create mode 100644 plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1 delete mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0.jar.sha1 create mode 100644 plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1 delete mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0.jar.sha1 create mode 100644 plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1 delete mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0.jar.sha1 create mode 100644 plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1 delete mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0.jar.sha1 create mode 100644 plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1 delete mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0.jar.sha1 create mode 100644 plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-analyzers-common-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-analyzers-common-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-backward-codecs-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-backward-codecs-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-core-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-core-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-grouping-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-grouping-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-highlighter-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-highlighter-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-join-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-join-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-memory-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-memory-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-misc-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-misc-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-queries-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-queries-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-queryparser-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-queryparser-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-sandbox-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-sandbox-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-spatial-extras-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial-extras-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-spatial3d-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-spatial3d-8.8.2.jar.sha1 delete mode 100644 server/licenses/lucene-suggest-8.7.0.jar.sha1 create mode 100644 server/licenses/lucene-suggest-8.8.2.jar.sha1 diff --git a/buildSrc/version.properties b/buildSrc/version.properties index bb0ec13f29a..3ce653aacfb 100644 --- a/buildSrc/version.properties +++ b/buildSrc/version.properties @@ -1,5 +1,5 @@ opensearch = 1.0.0 -lucene = 8.7.0 +lucene = 8.8.2 bundled_jdk_vendor = adoptopenjdk bundled_jdk = 15.0.1+9 diff --git a/modules/lang-expression/licenses/lucene-expressions-8.7.0.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.7.0.jar.sha1 deleted file mode 100644 index 5eb317c7605..00000000000 --- a/modules/lang-expression/licenses/lucene-expressions-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -71a442ef5ba4ff2543a69079b4a7f6f9d4f1f3d0 \ No newline at end of file diff --git a/modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1 b/modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1 new file mode 100644 index 00000000000..6ad83d9e604 --- /dev/null +++ b/modules/lang-expression/licenses/lucene-expressions-8.8.2.jar.sha1 @@ -0,0 +1 @@ +85c06b0d5b966e6d8a75dd568a6eb1a7c3b34bdd \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0.jar.sha1 deleted file mode 100644 index e88b4089dda..00000000000 --- a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -38f7d348e534c8d37482fcba75f4546df8e143af \ No newline at end of file diff --git a/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1 b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1 new file mode 100644 index 00000000000..1cd3a225ba1 --- /dev/null +++ b/plugins/analysis-icu/licenses/lucene-analyzers-icu-8.8.2.jar.sha1 @@ -0,0 +1 @@ +dcda3ccd38ca68e23ec57c8ca16d13553066672b \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0.jar.sha1 deleted file mode 100644 index faea773d1a4..00000000000 --- a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -f301e64abaf39204a5b5eb2c55c5152c7328e938 \ No newline at end of file diff --git a/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1 b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1 new file mode 100644 index 00000000000..20af2a97f28 --- /dev/null +++ b/plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-8.8.2.jar.sha1 @@ -0,0 +1 @@ +829b06a76ffaf0e551c5cda4a1e56990080ea6de \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0.jar.sha1 deleted file mode 100644 index bada6fd5ee5..00000000000 --- a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -3462eaa9b58e995c8e28060bf7ac2b054ca5e44e \ No newline at end of file diff --git a/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1 b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1 new file mode 100644 index 00000000000..a52d7d859bd --- /dev/null +++ b/plugins/analysis-nori/licenses/lucene-analyzers-nori-8.8.2.jar.sha1 @@ -0,0 +1 @@ +2c23e33c49fb255d96e0bf42ca7c58f030011727 \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0.jar.sha1 deleted file mode 100644 index 9ce0a10572b..00000000000 --- a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c5d438642d316cf2484d5a9f83ba8d35cdc28d5c \ No newline at end of file diff --git a/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1 b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1 new file mode 100644 index 00000000000..5443c1de42c --- /dev/null +++ b/plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-8.8.2.jar.sha1 @@ -0,0 +1 @@ +b0ccf32c2baa88815b89c157f19b264a24958180 \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0.jar.sha1 deleted file mode 100644 index 62471c41e84..00000000000 --- a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -6a58f7a9ac3562aecd2cfaf9a1097595e335be0e \ No newline at end of file diff --git a/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1 b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1 new file mode 100644 index 00000000000..cd9ce6b5143 --- /dev/null +++ b/plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-8.8.2.jar.sha1 @@ -0,0 +1 @@ +d7fb0c151dccc1954547acd3289cdf1369cda969 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0.jar.sha1 deleted file mode 100644 index c7eaf9999a2..00000000000 --- a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -22435907b6facd68e68300b329cc967137272861 \ No newline at end of file diff --git a/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1 b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1 new file mode 100644 index 00000000000..54d737721e2 --- /dev/null +++ b/plugins/analysis-stempel/licenses/lucene-analyzers-stempel-8.8.2.jar.sha1 @@ -0,0 +1 @@ +ca1b3379c99553c64d6b76271a5701f2bc2a2ce4 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0.jar.sha1 deleted file mode 100644 index 6237972e43d..00000000000 --- a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -a18c978ce5719b614445ca8f8075974df91d2631 \ No newline at end of file diff --git a/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1 b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1 new file mode 100644 index 00000000000..fb745852be9 --- /dev/null +++ b/plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-8.8.2.jar.sha1 @@ -0,0 +1 @@ +2070b7dd21682b4b9bd23f77146a73d0e3f3b05f \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.7.0.jar.sha1 b/server/licenses/lucene-analyzers-common-8.7.0.jar.sha1 deleted file mode 100644 index a32075aa4c9..00000000000 --- a/server/licenses/lucene-analyzers-common-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -fc219a023416a6fe6172f61d0d7bf64ed3645969 \ No newline at end of file diff --git a/server/licenses/lucene-analyzers-common-8.8.2.jar.sha1 b/server/licenses/lucene-analyzers-common-8.8.2.jar.sha1 new file mode 100644 index 00000000000..9dd818c4f24 --- /dev/null +++ b/server/licenses/lucene-analyzers-common-8.8.2.jar.sha1 @@ -0,0 +1 @@ +fa4b3c1d847f4bb91a3afaa990889dd164883d86 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.7.0.jar.sha1 b/server/licenses/lucene-backward-codecs-8.7.0.jar.sha1 deleted file mode 100644 index 79a6ab1661a..00000000000 --- a/server/licenses/lucene-backward-codecs-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -50c0493acdc9265d3a91365404e903bf408ab370 \ No newline at end of file diff --git a/server/licenses/lucene-backward-codecs-8.8.2.jar.sha1 b/server/licenses/lucene-backward-codecs-8.8.2.jar.sha1 new file mode 100644 index 00000000000..edec595d413 --- /dev/null +++ b/server/licenses/lucene-backward-codecs-8.8.2.jar.sha1 @@ -0,0 +1 @@ +f4a0475a1404d7be87e911f5c85f8b913ec9770b \ No newline at end of file diff --git a/server/licenses/lucene-core-8.7.0.jar.sha1 b/server/licenses/lucene-core-8.7.0.jar.sha1 deleted file mode 100644 index 25cb7117980..00000000000 --- a/server/licenses/lucene-core-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -ed64084a1502c2a6a411cbd9826131b81e0bf07f \ No newline at end of file diff --git a/server/licenses/lucene-core-8.8.2.jar.sha1 b/server/licenses/lucene-core-8.8.2.jar.sha1 new file mode 100644 index 00000000000..8254464770e --- /dev/null +++ b/server/licenses/lucene-core-8.8.2.jar.sha1 @@ -0,0 +1 @@ +e039fa90ee900c475bbc0943989ed5dccc556d3e \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.7.0.jar.sha1 b/server/licenses/lucene-grouping-8.7.0.jar.sha1 deleted file mode 100644 index 3b2aa39057d..00000000000 --- a/server/licenses/lucene-grouping-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -90a094913f46093745e779cce4d5548d03a824c9 \ No newline at end of file diff --git a/server/licenses/lucene-grouping-8.8.2.jar.sha1 b/server/licenses/lucene-grouping-8.8.2.jar.sha1 new file mode 100644 index 00000000000..9d66d2ff701 --- /dev/null +++ b/server/licenses/lucene-grouping-8.8.2.jar.sha1 @@ -0,0 +1 @@ +207ac15287656b6ce61ea921fefec3070a8dd99d \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.7.0.jar.sha1 b/server/licenses/lucene-highlighter-8.7.0.jar.sha1 deleted file mode 100644 index 512057c7477..00000000000 --- a/server/licenses/lucene-highlighter-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -84c88d872e8680ff334b25c2d85336a26b1c776a \ No newline at end of file diff --git a/server/licenses/lucene-highlighter-8.8.2.jar.sha1 b/server/licenses/lucene-highlighter-8.8.2.jar.sha1 new file mode 100644 index 00000000000..ee1068853ce --- /dev/null +++ b/server/licenses/lucene-highlighter-8.8.2.jar.sha1 @@ -0,0 +1 @@ +2674644580108335c08a25fd2e0f0c7dee010700 \ No newline at end of file diff --git a/server/licenses/lucene-join-8.7.0.jar.sha1 b/server/licenses/lucene-join-8.7.0.jar.sha1 deleted file mode 100644 index d74229599a6..00000000000 --- a/server/licenses/lucene-join-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -c2e04806a44227ff4c17530c6a0c8468d535b72e \ No newline at end of file diff --git a/server/licenses/lucene-join-8.8.2.jar.sha1 b/server/licenses/lucene-join-8.8.2.jar.sha1 new file mode 100644 index 00000000000..1ae40a66596 --- /dev/null +++ b/server/licenses/lucene-join-8.8.2.jar.sha1 @@ -0,0 +1 @@ +6eb956741169fe8a5643fa39b14aef2fc067d865 \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.7.0.jar.sha1 b/server/licenses/lucene-memory-8.7.0.jar.sha1 deleted file mode 100644 index 4c503e0dc00..00000000000 --- a/server/licenses/lucene-memory-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -52a346554a6160bfe1d32d93410d12bc2a795e0c \ No newline at end of file diff --git a/server/licenses/lucene-memory-8.8.2.jar.sha1 b/server/licenses/lucene-memory-8.8.2.jar.sha1 new file mode 100644 index 00000000000..2f6eb772a24 --- /dev/null +++ b/server/licenses/lucene-memory-8.8.2.jar.sha1 @@ -0,0 +1 @@ +63b6e8e862c898d263b941ffa55fe6c85bcdf2df \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.7.0.jar.sha1 b/server/licenses/lucene-misc-8.7.0.jar.sha1 deleted file mode 100644 index ee7eadcd2ea..00000000000 --- a/server/licenses/lucene-misc-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -e85fb21faa1ca9bd2c0f1283d9930adb058773e4 \ No newline at end of file diff --git a/server/licenses/lucene-misc-8.8.2.jar.sha1 b/server/licenses/lucene-misc-8.8.2.jar.sha1 new file mode 100644 index 00000000000..eabe707aa70 --- /dev/null +++ b/server/licenses/lucene-misc-8.8.2.jar.sha1 @@ -0,0 +1 @@ +a3a916c27c42ff6773c52d0be4045d4e5c00f7f8 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.7.0.jar.sha1 b/server/licenses/lucene-queries-8.7.0.jar.sha1 deleted file mode 100644 index e110c40a9a2..00000000000 --- a/server/licenses/lucene-queries-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -591f25f917a11143fbd9f4573468a0a8b5af21f2 \ No newline at end of file diff --git a/server/licenses/lucene-queries-8.8.2.jar.sha1 b/server/licenses/lucene-queries-8.8.2.jar.sha1 new file mode 100644 index 00000000000..460156c4b1b --- /dev/null +++ b/server/licenses/lucene-queries-8.8.2.jar.sha1 @@ -0,0 +1 @@ +1f324c3a6e1a7634bbada993b2c014d12d09cb83 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.7.0.jar.sha1 b/server/licenses/lucene-queryparser-8.7.0.jar.sha1 deleted file mode 100644 index f8bb7e39d48..00000000000 --- a/server/licenses/lucene-queryparser-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -66ae10cb549dea23346a4f5230de9f2f4880def4 \ No newline at end of file diff --git a/server/licenses/lucene-queryparser-8.8.2.jar.sha1 b/server/licenses/lucene-queryparser-8.8.2.jar.sha1 new file mode 100644 index 00000000000..35d68676948 --- /dev/null +++ b/server/licenses/lucene-queryparser-8.8.2.jar.sha1 @@ -0,0 +1 @@ +67c6580f8a5187fdb7f9eb2404bdecc4635b3328 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.7.0.jar.sha1 b/server/licenses/lucene-sandbox-8.7.0.jar.sha1 deleted file mode 100644 index 15115868177..00000000000 --- a/server/licenses/lucene-sandbox-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -b17d87dc165ddb58d3e6f8a6fe1622cb67bcd5c5 \ No newline at end of file diff --git a/server/licenses/lucene-sandbox-8.8.2.jar.sha1 b/server/licenses/lucene-sandbox-8.8.2.jar.sha1 new file mode 100644 index 00000000000..b1ff4925a5f --- /dev/null +++ b/server/licenses/lucene-sandbox-8.8.2.jar.sha1 @@ -0,0 +1 @@ +b674deb4957a64db5c8f4afb9127400f4e65c334 \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.7.0.jar.sha1 b/server/licenses/lucene-spatial-extras-8.7.0.jar.sha1 deleted file mode 100644 index 7fabc4705d9..00000000000 --- a/server/licenses/lucene-spatial-extras-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cbdb38220c0f1974385eb928c5a80ee9b440015f \ No newline at end of file diff --git a/server/licenses/lucene-spatial-extras-8.8.2.jar.sha1 b/server/licenses/lucene-spatial-extras-8.8.2.jar.sha1 new file mode 100644 index 00000000000..84dcad6d406 --- /dev/null +++ b/server/licenses/lucene-spatial-extras-8.8.2.jar.sha1 @@ -0,0 +1 @@ +227672b3b88c877b2df27608850ca15d834ec35c \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.7.0.jar.sha1 b/server/licenses/lucene-spatial3d-8.7.0.jar.sha1 deleted file mode 100644 index 043619bbbe7..00000000000 --- a/server/licenses/lucene-spatial3d-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -cd926effe54ef0eed0cfb6507cf71278f6f2c124 \ No newline at end of file diff --git a/server/licenses/lucene-spatial3d-8.8.2.jar.sha1 b/server/licenses/lucene-spatial3d-8.8.2.jar.sha1 new file mode 100644 index 00000000000..fe4ad693544 --- /dev/null +++ b/server/licenses/lucene-spatial3d-8.8.2.jar.sha1 @@ -0,0 +1 @@ +eb77b665a3b3e9ca6b3bb82c3342b843d9729d0d \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.7.0.jar.sha1 b/server/licenses/lucene-suggest-8.7.0.jar.sha1 deleted file mode 100644 index 97779a8b5e9..00000000000 --- a/server/licenses/lucene-suggest-8.7.0.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -07239e7962c99cfc976a21c4edb4b00c6d7db3b9 \ No newline at end of file diff --git a/server/licenses/lucene-suggest-8.8.2.jar.sha1 b/server/licenses/lucene-suggest-8.8.2.jar.sha1 new file mode 100644 index 00000000000..c25fdf198fd --- /dev/null +++ b/server/licenses/lucene-suggest-8.8.2.jar.sha1 @@ -0,0 +1 @@ +36fd5301e7143f0fcb0ba933f96148dc1e132b66 \ No newline at end of file diff --git a/server/src/main/java/org/opensearch/Version.java b/server/src/main/java/org/opensearch/Version.java index b86426c5736..09427ed55fa 100644 --- a/server/src/main/java/org/opensearch/Version.java +++ b/server/src/main/java/org/opensearch/Version.java @@ -70,7 +70,7 @@ public class Version implements Comparable, ToXContentFragment { public static final int V_EMPTY_ID = 0; public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST); - public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_7_0); + public static final Version V_1_0_0 = new Version(1000099, org.apache.lucene.util.Version.LUCENE_8_8_2); public static final Version CURRENT = V_1_0_0; public static Version readVersion(StreamInput in) throws IOException { diff --git a/server/src/main/java/org/opensearch/index/codec/CodecService.java b/server/src/main/java/org/opensearch/index/codec/CodecService.java index e1dd53c7716..afda55fea90 100644 --- a/server/src/main/java/org/opensearch/index/codec/CodecService.java +++ b/server/src/main/java/org/opensearch/index/codec/CodecService.java @@ -35,7 +35,7 @@ package org.opensearch.index.codec; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; import org.apache.lucene.codecs.lucene87.Lucene87Codec; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode; +import org.apache.lucene.codecs.lucene87.Lucene87Codec.Mode; import org.opensearch.common.Nullable; import org.opensearch.common.collect.MapBuilder; import org.opensearch.index.mapper.MapperService; diff --git a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java index 48b379ce46b..73e0f3e6b27 100644 --- a/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java +++ b/server/src/main/java/org/opensearch/index/codec/PerFieldMappingPostingFormatCodec.java @@ -34,8 +34,9 @@ package org.opensearch.index.codec; import org.apache.logging.log4j.Logger; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.DocValuesFormat; import org.apache.lucene.codecs.PostingsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat; +import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.codecs.lucene87.Lucene87Codec; import org.opensearch.common.lucene.Lucene; import org.opensearch.index.mapper.CompletionFieldMapper; @@ -53,13 +54,14 @@ import org.opensearch.index.mapper.MapperService; public class PerFieldMappingPostingFormatCodec extends Lucene87Codec { private final Logger logger; private final MapperService mapperService; + private final DocValuesFormat dvFormat = new Lucene80DocValuesFormat(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION); static { assert Codec.forName(Lucene.LATEST_CODEC).getClass().isAssignableFrom(PerFieldMappingPostingFormatCodec.class) : "PerFieldMappingPostingFormatCodec must subclass the latest " + "lucene codec: " + Lucene.LATEST_CODEC; } - public PerFieldMappingPostingFormatCodec(Lucene87StoredFieldsFormat.Mode compressionMode, MapperService mapperService, Logger logger) { + public PerFieldMappingPostingFormatCodec(Mode compressionMode, MapperService mapperService, Logger logger) { super(compressionMode); this.mapperService = mapperService; this.logger = logger; @@ -76,4 +78,8 @@ public class PerFieldMappingPostingFormatCodec extends Lucene87Codec { return super.getPostingsFormatForField(field); } + @Override + public DocValuesFormat getDocValuesFormatForField(String field) { + return dvFormat; + } } diff --git a/server/src/main/java/org/opensearch/index/query/VectorGeoShapeQueryProcessor.java b/server/src/main/java/org/opensearch/index/query/VectorGeoShapeQueryProcessor.java index 94c09e1d656..a3da23ab2b6 100644 --- a/server/src/main/java/org/opensearch/index/query/VectorGeoShapeQueryProcessor.java +++ b/server/src/main/java/org/opensearch/index/query/VectorGeoShapeQueryProcessor.java @@ -33,6 +33,7 @@ package org.opensearch.index.query; import org.apache.lucene.document.LatLonShape; +import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.geo.LatLonGeometry; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.Query; @@ -155,7 +156,12 @@ public class VectorGeoShapeQueryProcessor { @Override public Void visit(Point point) { if (point.isEmpty() == false) { - geometries.add(GeoShapeUtils.toLucenePoint(point)); + // points are a special "shape" case: for queries we need to quantize since the Lucene + // tessellator doesn't do anything with them. + // todo this is a sandy lucene experience so we should investigate fixing this upstream + double quantizedLat = GeoEncodingUtils.decodeLatitude(GeoEncodingUtils.encodeLatitude(point.getLat())); + double quantizedLon = GeoEncodingUtils.decodeLongitude(GeoEncodingUtils.encodeLongitude(point.getLon())); + geometries.add(new org.apache.lucene.geo.Point(quantizedLat, quantizedLon)); } return null; diff --git a/server/src/test/java/org/opensearch/index/codec/CodecTests.java b/server/src/test/java/org/opensearch/index/codec/CodecTests.java index 46035b8b518..b20e4bbbb28 100644 --- a/server/src/test/java/org/opensearch/index/codec/CodecTests.java +++ b/server/src/test/java/org/opensearch/index/codec/CodecTests.java @@ -34,15 +34,17 @@ package org.opensearch.index.codec; import org.apache.logging.log4j.LogManager; import org.apache.lucene.codecs.Codec; +import org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat; import org.apache.lucene.codecs.lucene87.Lucene87Codec; import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat; -import org.apache.lucene.codecs.lucene87.Lucene87StoredFieldsFormat.Mode; +import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.SegmentReader; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.LuceneTestCase.SuppressCodecs; import org.opensearch.common.settings.Settings; import org.opensearch.env.Environment; @@ -72,16 +74,37 @@ public class CodecTests extends OpenSearchTestCase { public void testDefault() throws Exception { Codec codec = createCodecService().codec("default"); - assertCompressionEquals(Mode.BEST_SPEED, codec); + assertDVCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_SPEED, codec); } public void testBestCompression() throws Exception { Codec codec = createCodecService().codec("best_compression"); - assertCompressionEquals(Mode.BEST_COMPRESSION, codec); + assertDVCompressionEquals(Lucene80DocValuesFormat.Mode.BEST_COMPRESSION, codec); + assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode.BEST_COMPRESSION, codec); + } + + private void assertDVCompressionEquals(Lucene80DocValuesFormat.Mode expected, Codec actual) throws Exception { + Directory dir = newDirectory(); + IndexWriterConfig iwc = newIndexWriterConfig(null); + iwc.setCodec(actual); + IndexWriter iw = new IndexWriter(dir, iwc); + Document doc = new Document(); + doc.add(new BinaryDocValuesField("foo", new BytesRef("aaa"))); + iw.addDocument(doc); + iw.commit(); + iw.close(); + DirectoryReader ir = DirectoryReader.open(dir); + SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); + String v = sr.getFieldInfos().fieldInfo("foo").getAttribute(Lucene80DocValuesFormat.MODE_KEY); + assertNotNull(v); + assertEquals(expected, Lucene80DocValuesFormat.Mode.valueOf(v)); + ir.close(); + dir.close(); } // write some docs with it, inspect .si to see this was the used compression - private void assertCompressionEquals(Mode expected, Codec actual) throws Exception { + private void assertStoredFieldsFormatCompressionEquals(Lucene87StoredFieldsFormat.Mode expected, Codec actual) throws Exception { Directory dir = newDirectory(); IndexWriterConfig iwc = newIndexWriterConfig(null); iwc.setCodec(actual); @@ -93,7 +116,7 @@ public class CodecTests extends OpenSearchTestCase { SegmentReader sr = (SegmentReader) ir.leaves().get(0).reader(); String v = sr.getSegmentInfo().info.getAttribute(Lucene87StoredFieldsFormat.MODE_KEY); assertNotNull(v); - assertEquals(expected, Mode.valueOf(v)); + assertEquals(expected, Lucene87StoredFieldsFormat.Mode.valueOf(v)); ir.close(); dir.close(); } diff --git a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java index 5299b5b63d1..8358e7cde65 100644 --- a/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java +++ b/test/framework/src/main/java/org/opensearch/indices/analysis/AnalysisFactoryTestCase.java @@ -219,6 +219,7 @@ public abstract class AnalysisFactoryTestCase extends OpenSearchTestCase { // TODO: these charfilters are not yet exposed: useful? // handling of zwnj for persian .put("persian", Void.class) + .put("cjkwidth", Void.class) .immutableMap(); /**