From 178f5a7a7e380d93b3928997ec21b3b718c3d29f Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 9 Feb 2024 23:02:42 +0100 Subject: [PATCH 01/24] Enable MemorySegment in MMapDirectory for Java 22+ and Vectorization (incubation) for exact Java 22 (#12706) --- gradle/generation/extract-jdk-apis.gradle | 2 +- lucene/CHANGES.txt | 11 +++++++++++ .../internal/vectorization/VectorizationProvider.java | 6 +++--- .../java/org/apache/lucene/store/MMapDirectory.java | 5 +---- .../apache/lucene/store/MemorySegmentIndexInput.java | 8 +++++++- .../lucene/store/MemorySegmentIndexInputProvider.java | 2 +- .../org/apache/lucene/store/TestMMapDirectory.java | 4 ++-- 7 files changed, 26 insertions(+), 12 deletions(-) diff --git a/gradle/generation/extract-jdk-apis.gradle b/gradle/generation/extract-jdk-apis.gradle index a24496db7fa..8ebdd6b1e33 100644 --- a/gradle/generation/extract-jdk-apis.gradle +++ b/gradle/generation/extract-jdk-apis.gradle @@ -20,7 +20,7 @@ def resources = scriptResources(buildscript) configure(rootProject) { ext { // also change this in extractor tool: ExtractForeignAPI - vectorIncubatorJavaVersions = [ JavaVersion.VERSION_20, JavaVersion.VERSION_21 ] as Set + vectorIncubatorJavaVersions = [ JavaVersion.VERSION_20, JavaVersion.VERSION_21, JavaVersion.VERSION_22 ] as Set } } diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index efadc8ba8e8..8856d9c7b80 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -200,6 +200,17 @@ New Features * GITHUB#12336: Index additional data per facet label in the taxonomy. (Shai Erera, Egor Potemkin, Mike McCandless, Stefan Vodita) +* GITHUB#12706: Add support for the final release of Java foreign memory API in Java 22 (and later). + Lucene's MMapDirectory will now mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) starting + from Java 19. Indexes closed while queries are running can no longer crash the JVM. + Support for vectorized implementations of VectorUtil based on jdk.incubator.vector APIs was added + for eactly Java 22. Therefore, applications started with command line parameter + "java --add-modules jdk.incubator.vector" will automatically use the new vectorized implementations + if running on a supported platform (Java 20/21/22 on x86 CPUs with AVX2 or later or ARM NEON CPUs). + This is an opt-in feature and requires explicit Java command line flag! When enabled, Lucene logs + a notice using java.util.logging. Please test thoroughly and report bugs/slowness to Lucene's mailing + list. (Uwe Schindler, Chris Hegarty) + Improvements --------------------- diff --git a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java index 73f1f918091..a1f07c6a043 100644 --- a/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java +++ b/lucene/core/src/java/org/apache/lucene/internal/vectorization/VectorizationProvider.java @@ -103,7 +103,7 @@ public abstract class VectorizationProvider { // visible for tests static VectorizationProvider lookup(boolean testMode) { final int runtimeVersion = Runtime.version().feature(); - if (runtimeVersion >= 20 && runtimeVersion <= 21) { + if (runtimeVersion >= 20 && runtimeVersion <= 22) { // is locale sane (only buggy in Java 20) if (isAffectedByJDK8301190()) { LOG.warning( @@ -169,9 +169,9 @@ public abstract class VectorizationProvider { } catch (ClassNotFoundException cnfe) { throw new LinkageError("PanamaVectorizationProvider is missing in Lucene JAR file", cnfe); } - } else if (runtimeVersion >= 22) { + } else if (runtimeVersion >= 23) { LOG.warning( - "You are running with Java 22 or later. To make full use of the Vector API, please update Apache Lucene."); + "You are running with Java 23 or later. To make full use of the Vector API, please update Apache Lucene."); } else if (lookupVectorModule().isPresent()) { LOG.warning( "Java vector incubator module was enabled by command line flags, but your Java version is too old: " diff --git a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java index af675a786fb..3bf8683266a 100644 --- a/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java @@ -346,7 +346,7 @@ public class MMapDirectory extends FSDirectory { } final var lookup = MethodHandles.lookup(); final int runtimeVersion = Runtime.version().feature(); - if (runtimeVersion >= 19 && runtimeVersion <= 21) { + if (runtimeVersion >= 19) { try { final var cls = lookup.findClass("org.apache.lucene.store.MemorySegmentIndexInputProvider"); // we use method handles, so we do not need to deal with setAccessible as we have private @@ -366,9 +366,6 @@ public class MMapDirectory extends FSDirectory { throw new LinkageError( "MemorySegmentIndexInputProvider is missing in Lucene JAR file", cnfe); } - } else if (runtimeVersion >= 22) { - LOG.warning( - "You are running with Java 22 or later. To make full use of MMapDirectory, please update Apache Lucene."); } return new MappedByteBufferIndexInputProvider(); } diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java index ba890d8056c..7d1e2572fdb 100644 --- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java +++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java @@ -108,10 +108,16 @@ abstract class MemorySegmentIndexInput extends IndexInput implements RandomAcces if (this.curSegment == null) { return new AlreadyClosedException("Already closed: " + this); } - // ISE can be thrown by MemorySegment and contains "closed" in message: + // in Java 22 or later we can check the isAlive status of all segments + // (see https://bugs.openjdk.org/browse/JDK-8310644): + if (Arrays.stream(segments).allMatch(s -> s.scope().isAlive()) == false) { + return new AlreadyClosedException("Already closed: " + this); + } + // fallback for Java 21: ISE can be thrown by MemorySegment and contains "closed" in message: if (e instanceof IllegalStateException && e.getMessage() != null && e.getMessage().contains("closed")) { + // the check is on message only, so preserve original cause for debugging: return new AlreadyClosedException("Already closed: " + this, e); } // otherwise rethrow unmodified NPE/ISE (as it possibly a bug with passing a null parameter to diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java index e994c2dddff..57c9b5275f0 100644 --- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java +++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInputProvider.java @@ -33,7 +33,7 @@ final class MemorySegmentIndexInputProvider implements MMapDirectory.MMapIndexIn public MemorySegmentIndexInputProvider() { var log = Logger.getLogger(getClass().getName()); log.info( - "Using MemorySegmentIndexInput with Java 21; to disable start with -D" + "Using MemorySegmentIndexInput with Java 21 or later; to disable start with -D" + MMapDirectory.ENABLE_MEMORY_SEGMENTS_SYSPROP + "=false"); } diff --git a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java index 8323ec4deb8..6be7e570a62 100644 --- a/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java +++ b/lucene/core/src/test/org/apache/lucene/store/TestMMapDirectory.java @@ -48,9 +48,9 @@ public class TestMMapDirectory extends BaseDirectoryTestCase { public void testCorrectImplementation() { final int runtimeVersion = Runtime.version().feature(); - if (runtimeVersion >= 19 && runtimeVersion <= 21) { + if (runtimeVersion >= 19) { assertTrue( - "on Java 19, 20, and 21 we should use MemorySegmentIndexInputProvider to create mmap IndexInputs", + "on Java 19 or later we should use MemorySegmentIndexInputProvider to create mmap IndexInputs", isMemorySegmentImpl()); } else { assertSame(MappedByteBufferIndexInputProvider.class, MMapDirectory.PROVIDER.getClass()); From 7e921e4ed9f19e7ed1f70726c0a7053e5b9d9df0 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Fri, 9 Feb 2024 23:15:05 +0100 Subject: [PATCH 02/24] fix typo in CHANGES.txt --- lucene/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 8856d9c7b80..bc8f6d61d40 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -204,7 +204,7 @@ New Features Lucene's MMapDirectory will now mmap Lucene indexes in chunks of 16 GiB (instead of 1 GiB) starting from Java 19. Indexes closed while queries are running can no longer crash the JVM. Support for vectorized implementations of VectorUtil based on jdk.incubator.vector APIs was added - for eactly Java 22. Therefore, applications started with command line parameter + for exactly Java 22. Therefore, applications started with command line parameter "java --add-modules jdk.incubator.vector" will automatically use the new vectorized implementations if running on a supported platform (Java 20/21/22 on x86 CPUs with AVX2 or later or ARM NEON CPUs). This is an opt-in feature and requires explicit Java command line flag! When enabled, Lucene logs From 7d99e6aafbdb32e1a6c94e10e41d58a1f790630a Mon Sep 17 00:00:00 2001 From: Dawid Weiss Date: Sun, 11 Feb 2024 11:14:58 +0100 Subject: [PATCH 03/24] Move changes and migration entries to Lucene 10.x since it seems there'll be no way to apply this patch in 9x? #12674 --- lucene/CHANGES.txt | 4 ++-- lucene/MIGRATE.md | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index bc8f6d61d40..64fcb356e6e 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -114,6 +114,8 @@ Improvements * GITHUB#12873: Expressions module now uses JEP 371 "Hidden Classes" with JEP 309 "Dynamic Class-File Constants" to implement Javascript expressions. (Uwe Schindler) +* GITHUB#11657, LUCENE-10621: Upgrade to OpenNLP 2.3.2. (Christine Poerschke, Eric Pugh) + Optimizations --------------------- @@ -230,8 +232,6 @@ Improvements Tests are running with random byte order to ensure that the order does not affect correctness of code. Native order was enabled for LZ4 compression. (Uwe Schindler) -* GITHUB#11657, LUCENE-10621: Upgrade to OpenNLP 2.3.2. (Christine Poerschke, Eric Pugh) - Optimizations --------------------- diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md index 8a72c58b097..20606df9aac 100644 --- a/lucene/MIGRATE.md +++ b/lucene/MIGRATE.md @@ -19,6 +19,10 @@ ## Migration from Lucene 9.x to Lucene 10.0 +### OpenNLP dependency upgrade + +[Apache OpenNLP](https://opennlp.apache.org) 2.x opens the door to accessing various models via the ONNX runtime. To migrate you will need to update any deprecated OpenNLP methods that you may be using and be running on Java 17. + ### IndexWriter requires a parent document field in order to use index sorting with document blocks (GITHUB#12829) For indices newly created as of 10.0.0 onwards, IndexWriter preserves document blocks indexed via @@ -149,10 +153,6 @@ offsets, making positional queries and highlighters possible for fields tokenize ## Migration from Lucene 9.9 to Lucene 9.10 -### OpenNLP dependency upgrade - -[Apache OpenNLP](https://opennlp.apache.org) 2.x opens the door to accessing various models via the ONNX runtime. To migrate you will need to update any deprecated OpenNLP methods that you may be using and be running on Java 17. - ## Migration from Lucene 9.0 to Lucene 9.1 ### Test framework package migration and module (LUCENE-10301) From 8bf10130e95c7b1b3da16606be0bf872c28fb6ed Mon Sep 17 00:00:00 2001 From: Christine Poerschke Date: Mon, 12 Feb 2024 10:17:41 +0000 Subject: [PATCH 04/24] lucene/MIGRATE.md update: remove empty 9.9 to 9.10 section --- lucene/MIGRATE.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/lucene/MIGRATE.md b/lucene/MIGRATE.md index 20606df9aac..ded95efb90a 100644 --- a/lucene/MIGRATE.md +++ b/lucene/MIGRATE.md @@ -151,8 +151,6 @@ may throw `IOException` on index problems, bubbling up unexpectedly to the calle `(Reverse)PathHierarchyTokenizer` now produces sequential (instead of overlapping) tokens with accurate offsets, making positional queries and highlighters possible for fields tokenized with this tokenizer. -## Migration from Lucene 9.9 to Lucene 9.10 - ## Migration from Lucene 9.0 to Lucene 9.1 ### Test framework package migration and module (LUCENE-10301) From 314c553bdce22bc1bc08e46b218691575c2287e0 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 12 Feb 2024 14:36:17 +0100 Subject: [PATCH 05/24] Add next major version 9.11.0 --- lucene/CHANGES.txt | 26 +++++++++++++++++++ .../java/org/apache/lucene/util/Version.java | 10 ++++++- 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 64fcb356e6e..3759d31f1ba 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -178,6 +178,32 @@ Other * GITHUB#13001: Put Thread#sleep() on the list of forbidden APIs. (Shubham Chaudhary) +======================== Lucene 9.11.0 ======================= + +API Changes +--------------------- +(No changes) + +New Features +--------------------- +(No changes) + +Improvements +--------------------- +(No changes) + +Optimizations +--------------------- +(No changes) + +Bug Fixes +--------------------- +(No changes) + +Other +--------------------- +(No changes) + ======================== Lucene 9.10.0 ======================= API Changes diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java index 7baf53b6b32..5e80568174e 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Version.java +++ b/lucene/core/src/java/org/apache/lucene/util/Version.java @@ -133,9 +133,17 @@ public final class Version { /** * Match settings and bugs in Lucene's 9.10.0 release. * + * @deprecated (9.11.0) Use latest + */ + @Deprecated + @Deprecated public static final Version LUCENE_9_10_0 = new Version(9, 10, 0); + + /** + * Match settings and bugs in Lucene's 9.11.0 release. * @deprecated Use latest */ - @Deprecated public static final Version LUCENE_9_10_0 = new Version(9, 10, 0); + @Deprecated + public static final Version LUCENE_9_11_0 = new Version(9, 11, 0); /** * Match settings and bugs in Lucene's 10.0.0 release. From 027240f522e32001d2daf6a622aeb7fbe87f8869 Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Mon, 12 Feb 2024 17:01:47 +0100 Subject: [PATCH 06/24] Tidy after version bump. --- lucene/core/src/java/org/apache/lucene/util/Version.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/util/Version.java b/lucene/core/src/java/org/apache/lucene/util/Version.java index 5e80568174e..085f9094c27 100644 --- a/lucene/core/src/java/org/apache/lucene/util/Version.java +++ b/lucene/core/src/java/org/apache/lucene/util/Version.java @@ -135,15 +135,14 @@ public final class Version { * * @deprecated (9.11.0) Use latest */ - @Deprecated @Deprecated public static final Version LUCENE_9_10_0 = new Version(9, 10, 0); /** * Match settings and bugs in Lucene's 9.11.0 release. + * * @deprecated Use latest */ - @Deprecated - public static final Version LUCENE_9_11_0 = new Version(9, 11, 0); + @Deprecated public static final Version LUCENE_9_11_0 = new Version(9, 11, 0); /** * Match settings and bugs in Lucene's 10.0.0 release. From f6e065dc31cbd05e76e47facecf53efc82f93d90 Mon Sep 17 00:00:00 2001 From: Zhang Chao <80152403@qq.com> Date: Tue, 13 Feb 2024 10:10:44 +0800 Subject: [PATCH 07/24] Fix test failure for TestDocumentsWriterDeleteQueue.testUpdateDeleteSlices (#13089) --- .../org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java b/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java index 95368c5c6ad..041f00c113a 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestDocumentsWriterDeleteQueue.java @@ -67,7 +67,7 @@ public class TestDocumentsWriterDeleteQueue extends LuceneTestCase { assertAllBetween(last2, j, bd2, ids); last2 = j + 1; } - assertEquals(j + 1, queue.numGlobalTermDeletes()); + assertEquals(uniqueValues.size(), queue.numGlobalTermDeletes()); } assertEquals(uniqueValues, bd1.deleteTerms.keySet()); assertEquals(uniqueValues, bd2.deleteTerms.keySet()); From d7e0a7f0e48c518a731d96fcc431871253e0beaf Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 13 Feb 2024 10:48:30 +0100 Subject: [PATCH 08/24] Only track released versions in `oldVersions`. (#13096) --- .../BackwardsCompatibilityTestBase.java | 55 +++++++++++++++---- 1 file changed, 45 insertions(+), 10 deletions(-) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java index ba5a395d310..5081f164dfe 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java @@ -27,6 +27,7 @@ import java.nio.file.Paths; import java.text.ParseException; import java.util.ArrayList; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Locale; import java.util.Set; @@ -61,8 +62,7 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase { "8.7.0", "8.7.0", "8.8.0", "8.8.0", "8.8.1", "8.8.1", "8.8.2", "8.8.2", "8.9.0", "8.9.0", "8.10.0", "8.10.0", "8.10.1", "8.10.1", "8.11.0", "8.11.0", "8.11.1", "8.11.1", "8.11.2", "8.11.2", "8.11.3", "8.11.3", "9.0.0", "9.1.0", "9.2.0", "9.3.0", "9.4.0", "9.4.1", - "9.4.2", "9.5.0", "9.6.0", "9.7.0", "9.8.0", "9.9.0", "9.9.1", "9.9.2", "9.10.0", - "10.0.0", + "9.4.2", "9.5.0", "9.6.0", "9.7.0", "9.8.0", "9.9.0", "9.9.1", "9.9.2" }; Set binaryVersions = new HashSet<>(); @@ -75,8 +75,8 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase { throw new RuntimeException(ex); } } - List allCurrentVersions = getAllCurrentVersions(); - for (Version version : allCurrentVersions) { + + for (Version version : getAllCurrentReleasedVersions()) { // make sure we never miss a version. assertTrue("Version: " + version + " missing", binaryVersions.remove(version)); } @@ -181,19 +181,54 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase { return versions; } + private static List getAllCurrentReleasedVersions() { + List currentReleasedVersions = getAllCurrentVersions(); + + // The latest version from the current major is always under development. + assertTrue(currentReleasedVersions.remove(Version.LATEST)); + if (Version.LATEST.minor == 0 && Version.LATEST.bugfix == 0) { + // If the current branch points to the next major, then the latest minor from the previous + // major is also under development. + assertTrue(currentReleasedVersions.remove(LATEST_PREVIOUS_MAJOR)); + } + + // In addition to those, we may need to remove one more version in case a release is in + // progress, and the version constant has been added but backward-compatibility indexes have not + // been checked in yet. + List missingVersions = new ArrayList<>(); + for (Iterator it = currentReleasedVersions.iterator(); it.hasNext(); ) { + Version version = it.next(); + String indexName = String.format(Locale.ROOT, "index.%s-cfs.zip", version); + if (TestAncientIndicesCompatibility.class.getResource(indexName) == null) { + missingVersions.add(version); + it.remove(); + } + } + + if (missingVersions.size() > 1) { + throw new AssertionError( + "More than one version is missing backward-compatibility data: " + missingVersions); + } + return currentReleasedVersions; + } + + /** Get all versions that are released, plus the latest version which is unreleased. */ + public static List getAllCurrentReleasedVersionsAndCurrent() { + List versions = new ArrayList<>(getAllCurrentReleasedVersions()); + versions.add(Version.LATEST); + return versions; + } + public static Iterable allVersion(String name, String... suffixes) { List patterns = new ArrayList<>(); for (String suffix : suffixes) { patterns.add(createPattern(name, suffix)); } List versionAndPatterns = new ArrayList<>(); - List versionList = getAllCurrentVersions(); + List versionList = getAllCurrentReleasedVersionsAndCurrent(); for (Version v : versionList) { - if (v.equals(LATEST_PREVIOUS_MAJOR) - == false) { // the latest prev-major has not yet been released - for (Object p : patterns) { - versionAndPatterns.add(new Object[] {v, p}); - } + for (Object p : patterns) { + versionAndPatterns.add(new Object[] {v, p}); } } return versionAndPatterns; From 6f026d7c36c43f4571af8ffdf05caec83bf941db Mon Sep 17 00:00:00 2001 From: Adrien Grand Date: Tue, 13 Feb 2024 10:55:46 +0100 Subject: [PATCH 09/24] Fix logic to always exclude latest minor from previous major. --- .../backward_index/BackwardsCompatibilityTestBase.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java index 5081f164dfe..b238f952132 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/BackwardsCompatibilityTestBase.java @@ -186,11 +186,8 @@ public abstract class BackwardsCompatibilityTestBase extends LuceneTestCase { // The latest version from the current major is always under development. assertTrue(currentReleasedVersions.remove(Version.LATEST)); - if (Version.LATEST.minor == 0 && Version.LATEST.bugfix == 0) { - // If the current branch points to the next major, then the latest minor from the previous - // major is also under development. - assertTrue(currentReleasedVersions.remove(LATEST_PREVIOUS_MAJOR)); - } + // The latest minor from the previous major is also under development. + assertTrue(currentReleasedVersions.remove(LATEST_PREVIOUS_MAJOR)); // In addition to those, we may need to remove one more version in case a release is in // progress, and the version constant has been added but backward-compatibility indexes have not From c42380cbd0bfeef8052ab1211678a59675778545 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Tue, 13 Feb 2024 11:57:10 +0100 Subject: [PATCH 10/24] Enable parent field in sorted bwc tests (#13067) This enables the optional parent field in BWC tests from 9.10 on. This will need to be forward ported to main branch where the parent field is required to these tests since they add document blocks during tests. --- .../TestIndexSortBackwardsCompatibility.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexSortBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexSortBackwardsCompatibility.java index e1f061ec3d3..c57f319213f 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexSortBackwardsCompatibility.java +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestIndexSortBackwardsCompatibility.java @@ -55,6 +55,8 @@ public class TestIndexSortBackwardsCompatibility extends BackwardsCompatibilityT static final String INDEX_NAME = "sorted"; static final String SUFFIX = ""; + private static final Version FIRST_PARENT_DOC_VERSION = Version.LUCENE_9_10_0; + private static final String PARENT_FIELD_NAME = "___parent"; public TestIndexSortBackwardsCompatibility(Version version, String pattern) { super(version, pattern); @@ -79,8 +81,8 @@ public class TestIndexSortBackwardsCompatibility extends BackwardsCompatibilityT .setOpenMode(IndexWriterConfig.OpenMode.APPEND) .setIndexSort(sort) .setMergePolicy(newLogMergePolicy()); - if (this.version.onOrAfter(Version.LUCENE_10_0_0)) { - indexWriterConfig.setParentField("___parent"); + if (this.version.onOrAfter(FIRST_PARENT_DOC_VERSION)) { + indexWriterConfig.setParentField(PARENT_FIELD_NAME); } // open writer try (IndexWriter writer = new IndexWriter(directory, indexWriterConfig)) { @@ -89,7 +91,10 @@ public class TestIndexSortBackwardsCompatibility extends BackwardsCompatibilityT Document child = new Document(); child.add(new StringField("relation", "child", Field.Store.NO)); child.add(new StringField("bid", "" + i, Field.Store.NO)); - child.add(new NumericDocValuesField("dateDV", i)); + if (version.onOrAfter(FIRST_PARENT_DOC_VERSION) + == false) { // only add this to earlier versions + child.add(new NumericDocValuesField("dateDV", i)); + } Document parent = new Document(); parent.add(new StringField("relation", "parent", Field.Store.NO)); parent.add(new StringField("bid", "" + i, Field.Store.NO)); @@ -158,6 +163,7 @@ public class TestIndexSortBackwardsCompatibility extends BackwardsCompatibilityT conf.setUseCompoundFile(false); conf.setCodec(TestUtil.getDefaultCodec()); conf.setParentField("___parent"); + conf.setParentField(PARENT_FIELD_NAME); conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true))); IndexWriter writer = new IndexWriter(directory, conf); LineFileDocs docs = new LineFileDocs(new Random(0)); From 2639aea5dd1f5bf21072fe71ddd081238d4220c4 Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Tue, 13 Feb 2024 14:19:33 +0100 Subject: [PATCH 11/24] Change the links to Lucene's homepage in the DOAP file to https:// (#13097) --- dev-tools/doap/lucene.rdf | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/dev-tools/doap/lucene.rdf b/dev-tools/doap/lucene.rdf index b0c5039b4c5..72a976a94fa 100644 --- a/dev-tools/doap/lucene.rdf +++ b/dev-tools/doap/lucene.rdf @@ -23,23 +23,23 @@ xmlns:asfext="http://projects.apache.org/ns/asfext#" xmlns:foaf="http://xmlns.com/foaf/0.1/"> - + 2001-09-01 Apache Lucene Core - - + + Apache Lucene is a high-performance, full-featured text search engine library Apache Lucene is a high-performance, full-featured text search engine library written entirely in Java. It is a technology suitable for nearly any application that requires full-text search, especially cross-platform. - - + + Java