diff --git a/dev-tools/doap/lucene.rdf b/dev-tools/doap/lucene.rdf index b5487a008f2..5c6c108274f 100644 --- a/dev-tools/doap/lucene.rdf +++ b/dev-tools/doap/lucene.rdf @@ -67,6 +67,13 @@ + + + lucene-10.1.0 + 2024-12-20 + 10.1.0 + + lucene-10.0.0 diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 99347d8984c..6b01271c46b 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -182,6 +182,9 @@ Bug Fixes * GITHUB#13990: Added filter to the toString() method of Knn[Float|Byte]VectorQuery and DiversifyingChildren[Float|Byte]KnnVectorQuery. (Viswanath Kuchibhotla) * GITHUB#13819: Prevent flattening of ordered and unordered interval sources (Jim Ferenczi) +* GITHUB#14008: Counts provided by taxonomy facets in addition to another aggregation are now returned together with + their corresponding ordinals. (Paul King) +* GITHUB#14027: Make SegmentInfos#readCommit(Directory, String, int) public (Luca Cavanna) Build --------------------- @@ -192,16 +195,6 @@ Other --------------------- * GITHUB#13982: Remove duplicate test code. (Lu Xugang) -======================== Lucene 10.0.1 ======================= - -Bug Fixes ---------------------- - -* GITHUB#14008: Counts provided by taxonomy facets in addition to another aggregation are now returned together with - their corresponding ordinals. (Paul King) - -* GITHUB#14027: Make SegmentInfos#readCommit(Directory, String, int) public (Luca Cavanna) - ======================= Lucene 10.0.0 ======================= API Changes diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.10.1.0-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.10.1.0-cfs.zip new file mode 100644 index 00000000000..b9beaa47658 Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.10.1.0-cfs.zip differ diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.10.1.0-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.10.1.0-nocfs.zip new file mode 100644 index 00000000000..1b36dcb4ac3 Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.10.1.0-nocfs.zip differ diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/int7_hnsw.10.1.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/int7_hnsw.10.1.0.zip new file mode 100644 index 00000000000..350cf697f73 Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/int7_hnsw.10.1.0.zip differ diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.10.1.0.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.10.1.0.zip new file mode 100644 index 00000000000..882bbf9a533 Binary files /dev/null and b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.10.1.0.zip differ diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.12.1-cfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.9.12.1-cfs.zip similarity index 100% rename from lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.12.1-cfs.zip rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.9.12.1-cfs.zip diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.12.1-nocfs.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.9.12.1-nocfs.zip similarity index 100% rename from lucene/backward-codecs/src/test/org/apache/lucene/backward_index/index.9.12.1-nocfs.zip rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.9.12.1-nocfs.zip diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/int7_hnsw.9.12.1.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.int7_hnsw.9.12.1.zip similarity index 100% rename from lucene/backward-codecs/src/test/org/apache/lucene/backward_index/int7_hnsw.9.12.1.zip rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.int7_hnsw.9.12.1.zip diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.9.12.1.zip b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.sorted.9.12.1.zip similarity index 100% rename from lucene/backward-codecs/src/test/org/apache/lucene/backward_index/sorted.9.12.1.zip rename to lucene/backward-codecs/src/test/org/apache/lucene/backward_index/unsupported.sorted.9.12.1.zip diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/versions.txt b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/versions.txt index f4015e5728e..850b6da9bd0 100644 --- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/versions.txt +++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/versions.txt @@ -17,4 +17,5 @@ 9.11.1 9.12.0 9.12.1 -10.0.0 \ No newline at end of file +10.0.0 +10.1.0 diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java index 6e8bbf81966..e3074a96d88 100644 --- a/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java +++ b/lucene/core/src/java/org/apache/lucene/search/IndexSearcher.java @@ -845,7 +845,9 @@ public class IndexSearcher { scorer = new TimeLimitingBulkScorer(scorer, queryTimeout); } try { - scorer.score(leafCollector, ctx.reader().getLiveDocs(), minDocId, maxDocId); + // Optimize for the case when live docs are stored in a FixedBitSet. + Bits acceptDocs = ScorerUtil.likelyFixedBitSet(ctx.reader().getLiveDocs()); + scorer.score(leafCollector, acceptDocs, minDocId, maxDocId); } catch ( @SuppressWarnings("unused") CollectionTerminatedException e) { diff --git a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java index 1154dc1b154..2081f9ac717 100644 --- a/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java +++ b/lucene/core/src/java/org/apache/lucene/search/ScorerUtil.java @@ -30,7 +30,9 @@ import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.store.ByteBuffersDirectory; import org.apache.lucene.store.Directory; +import org.apache.lucene.util.Bits; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.FixedBitSet; import org.apache.lucene.util.PriorityQueue; /** Util class for Scorer related methods */ @@ -108,4 +110,40 @@ class ScorerUtil { } return scorable; } + + /** + * Optimize {@link Bits} representing the set of accepted documents for the case when it is likely + * implemented via a {@link FixedBitSet}. This helps make calls to {@link Bits#get(int)} + * inlinable, which in-turn helps speed up query evaluation. This is especially helpful as + * inlining will sometimes enable auto-vectorizing shifts and masks that are done in {@link + * FixedBitSet#get(int)}. + */ + static Bits likelyFixedBitSet(Bits acceptDocs) { + if (acceptDocs instanceof FixedBitSet) { + return acceptDocs; + } else if (acceptDocs != null) { + return new FilterBits(acceptDocs); + } else { + return null; + } + } + + private static class FilterBits implements Bits { + + private final Bits in; + + FilterBits(Bits in) { + this.in = in; + } + + @Override + public boolean get(int index) { + return in.get(index); + } + + @Override + public int length() { + return in.length(); + } + } } diff --git a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java index 9273f7c5a81..1369aa5e3f4 100644 --- a/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java +++ b/lucene/core/src/java21/org/apache/lucene/internal/vectorization/PanamaVectorUtilSupport.java @@ -771,6 +771,10 @@ final class PanamaVectorUtilSupport implements VectorUtilSupport { @Override public int findNextGEQ(int[] buffer, int target, int from, int to) { if (ENABLE_FIND_NEXT_GEQ_VECTOR_OPTO) { + // This effectively implements the V1 intersection algorithm from + // D. Lemire, L. Boytsov, N. Kurz SIMD Compression and the Intersection of Sorted Integers + // with T = INT_SPECIES.length(), ie. T=8 with AVX2 and T=16 with AVX-512 + // https://arxiv.org/pdf/1401.6399 for (; from + INT_SPECIES.length() < to; from += INT_SPECIES.length() + 1) { if (buffer[from + INT_SPECIES.length()] >= target) { IntVector vector = IntVector.fromArray(INT_SPECIES, buffer, from);