Upgrade to lucene-7.2.0-snapshot-8c94404. (#27496)

The main highlight of this new snapshot is that it introduces the opportunity
for queries to opt out of caching. In case a query opts out of caching, not only
will it never be cached, but also no compound query that wraps it will be
cached.
This commit is contained in:
Adrien Grand 2017-11-28 14:52:42 +01:00 committed by GitHub
parent cb1204774b
commit 996990ad1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
72 changed files with 170 additions and 85 deletions

View File

@ -1,5 +1,5 @@
elasticsearch = 7.0.0-alpha1
lucene = 7.1.0
lucene = 7.2.0-snapshot-8c94404
# optional dependencies
spatial4j = 0.6

View File

@ -1 +0,0 @@
a508bf6b580471ee568dab7d2acfedfa5aadce70

View File

@ -0,0 +1 @@
4c515e5152e6938129a5e97c5afb5b3b360faed3

View File

@ -1 +0,0 @@
804a7ce82bba3d085733486bfde4846ecb77ce01

View File

@ -0,0 +1 @@
406c6cc0f8c2a47d42a1e343eaf2ad939fee905c

View File

@ -1 +0,0 @@
dd291b7ebf4845483895724d2562214dc7f40049

View File

@ -0,0 +1 @@
4c93f7fbc6e0caf87f7948b8481d80e0167133bf

View File

@ -1 +0,0 @@
0732d16c16421fca058a2a07ca4081ec7696365b

View File

@ -0,0 +1 @@
b078ca50c6d579085c7755b4fd8de60711964dcc

View File

@ -1 +0,0 @@
596550daabae765ad685112e0fe7c4f0fdfccb3f

View File

@ -0,0 +1 @@
fc5e61c8879f22b65ee053f1665bc9f13af79c1d

View File

@ -1 +0,0 @@
5f26dd64c195258a81175772ef7fe105e7d60a26

View File

@ -0,0 +1 @@
9a10839d3dfe7b369f0af8a78a630ee4d82e678e

View File

@ -1 +0,0 @@
3ef64c58d0c09ca40d848efa96b585b7476271f2

View File

@ -0,0 +1 @@
d45f2f51cf6f47a66ecafddecb83c1e08eb4061f

View File

@ -1 +0,0 @@
1496ee5fa62206ee5ddf51042a340d6a9ee3b5de

View File

@ -0,0 +1 @@
19cb7362be57104ad891259060af80fb4679e92c

View File

@ -1 +0,0 @@
1554920ab207a3245fa408d022a5c90ad3a1fea3

View File

@ -0,0 +1 @@
ae24737048d95f56d0099fea77498324412eef50

View File

@ -1 +0,0 @@
5767c15c5ee97926829fd8a4337e434fa95f3c08

View File

@ -0,0 +1 @@
a9d3422c9a72106026c19a8f76a4f4e62159ff5c

View File

@ -1 +0,0 @@
691f7b9ac05f3ad2ac7e80733ef70247904bd3ae

View File

@ -0,0 +1 @@
66433006587ede3e01899fd6f5e55c8378032c2f

View File

@ -1 +0,0 @@
6c64c04d802badb800516a8a574cb993929c3805

View File

@ -0,0 +1 @@
b6b3082ba845f7bd41641b015624f46d4f20afb6

View File

@ -1 +0,0 @@
3f1bc1aada8f06b176b782da24b9d7ad9641c41a

View File

@ -0,0 +1 @@
7757cac49cb3e9f1a219346ce95fb80f61f7090e

View File

@ -1 +0,0 @@
8ded650aed23efb775f17be496e3e3870214e23b

View File

@ -0,0 +1 @@
92991fdcd185883050d9530ccc0d863b7a08e99c

View File

@ -1 +0,0 @@
8d0ed1589ebdccf34e888c6efc0134a13a238c85

View File

@ -0,0 +1 @@
fb6a94b833a23a17e3721ea2f9679ad770dec48b

View File

@ -93,6 +93,15 @@ public final class MinDocQuery extends Query {
final DocIdSetIterator disi = new MinDocIterator(segmentMinDoc, maxDoc);
return new ConstantScoreScorer(this, score(), disi);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// Let's not cache this query, the cached iterator would use more memory
// and be slower anyway.
// Also, matches in a given segment depend on the other segments, which
// makes it a bad candidate for per-segment caching.
return false;
}
};
}

View File

@ -35,9 +35,7 @@ import org.apache.lucene.search.SortField;
import org.apache.lucene.search.Weight;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
/**
@ -90,6 +88,14 @@ public class SearchAfterSortedDocQuery extends Query {
final DocIdSetIterator disi = new MinDocQuery.MinDocIterator(firstDoc, maxDoc);
return new ConstantScoreScorer(this, score(), disi);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// If the sort order includes _doc, then the matches in a segment
// may depend on other segments, which makes this query a bad
// candidate for caching
return false;
}
};
}

View File

@ -137,7 +137,7 @@ public class Version implements Comparable<Version> {
public static final Version V_6_2_0 = new Version(V_6_2_0_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
public static final int V_7_0_0_alpha1_ID = 7000001;
public static final Version V_7_0_0_alpha1 =
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_1_0);
new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_2_0);
public static final Version CURRENT = V_7_0_0_alpha1;
static {

View File

@ -693,10 +693,6 @@ public class Lucene {
throw new IllegalStateException(message);
}
@Override
public int freq() throws IOException {
throw new IllegalStateException(message);
}
@Override
public int docID() {
throw new IllegalStateException(message);
}

View File

@ -333,6 +333,13 @@ public class FunctionScoreQuery extends Query {
}
return expl;
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// If minScore is not null, then matches depend on statistics of the
// top-level reader.
return minScore == null;
}
}
static class FunctionFactorScorer extends FilterScorer {

View File

@ -59,11 +59,6 @@ final class MinScoreScorer extends Scorer {
return in.score();
}
@Override
public int freq() throws IOException {
return in.freq();
}
@Override
public DocIdSetIterator iterator() {
return TwoPhaseIterator.asDocIdSetIterator(twoPhaseIterator());

View File

@ -50,11 +50,6 @@ public class ScriptScoreFunction extends ScoreFunction {
return score;
}
@Override
public int freq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public DocIdSetIterator iterator() {
throw new UnsupportedOperationException();

View File

@ -153,23 +153,17 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
@Override
public boolean equals(Object obj) {
// TODO: Do this if/when we can assume scripts are pure functions
// and they have a reliable equals impl
/*if (this == obj)
return true;
if (sameClassAs(obj) == false)
return false;
ScriptQuery other = (ScriptQuery) obj;
return Objects.equals(script, other.script);*/
return this == obj;
return Objects.equals(script, other.script);
}
@Override
public int hashCode() {
// TODO: Do this if/when we can assume scripts are pure functions
// and they have a reliable equals impl
// return Objects.hash(classHash(), script);
return System.identityHashCode(this);
int h = classHash();
h = 31 * h + script.hashCode();
return h;
}
@Override
@ -196,6 +190,14 @@ public class ScriptQueryBuilder extends AbstractQueryBuilder<ScriptQueryBuilder>
};
return new ConstantScoreScorer(this, score(), twoPhase);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// TODO: Change this to true when we can assume that scripts are pure functions
// ie. the return value is always the same given the same conditions and may not
// depend on the current timestamp, other documents, etc.
return false;
}
};
}
}

View File

@ -26,6 +26,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanQuery;
import org.apache.lucene.search.CoveringQuery;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.LongValues;
import org.apache.lucene.search.LongValuesSource;
import org.apache.lucene.search.Query;
@ -290,13 +291,18 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder<TermsSetQue
@Override
public int hashCode() {
// CoveringQuery with this field value source cannot be cachable
return System.identityHashCode(this);
int h = getClass().hashCode();
h = 31 * h + script.hashCode();
return h;
}
@Override
public boolean equals(Object obj) {
return this == obj;
if (obj == null || getClass() != obj.getClass()) {
return false;
}
ScriptLongValueSource that = (ScriptLongValueSource) obj;
return Objects.equals(script, that.script);
}
@Override
@ -304,6 +310,19 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder<TermsSetQue
return "script(" + script.toString() + ")";
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// TODO: Change this to true when we can assume that scripts are pure functions
// ie. the return value is always the same given the same conditions and may not
// depend on the current timestamp, other documents, etc.
return false;
}
@Override
public LongValuesSource rewrite(IndexSearcher searcher) throws IOException {
return this;
}
}
// Forked from LongValuesSource.FieldValuesSource and changed getValues() method to always use sorted numeric
@ -364,6 +383,16 @@ public final class TermsSetQueryBuilder extends AbstractQueryBuilder<TermsSetQue
public boolean needsScores() {
return false;
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return true;
}
@Override
public LongValuesSource rewrite(IndexSearcher searcher) throws IOException {
return this;
}
}
}

View File

@ -157,6 +157,13 @@ final class ShardSplittingQuery extends Query {
return new ConstantScoreScorer(this, score(), new BitSetIterator(bitSet, bitSet.length()));
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// This is not a regular query, let's not cache it. It wouldn't help
// anyway.
return false;
}
};
}

View File

@ -157,6 +157,11 @@ public class IndicesQueryCache extends AbstractComponent implements QueryCache,
shardKeyMap.add(context.reader());
return in.bulkScorer(context);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return in.isCacheable(ctx);
}
}
/** Clear all entries that belong to the given index. */

View File

@ -259,11 +259,6 @@ public class BestDocsDeferringCollector extends DeferringBucketCollector impleme
return currentScore;
}
@Override
public int freq() throws IOException {
throw new ElasticsearchException("This caching scorer implementation only implements score() and docID()");
}
@Override
public int docID() {
return currentDocId;

View File

@ -170,6 +170,11 @@ public class ContextIndexSearcher extends IndexSearcher implements Releasable {
throw new UnsupportedOperationException();
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
throw new UnsupportedOperationException();
}
@Override
public BulkScorer bulkScorer(LeafReaderContext context) throws IOException {
BulkScorer in = weight.bulkScorer(context);

View File

@ -63,11 +63,6 @@ final class ProfileScorer extends Scorer {
}
}
@Override
public int freq() throws IOException {
return scorer.freq();
}
@Override
public Weight getWeight() {
return profileWeight;

View File

@ -118,4 +118,9 @@ public final class ProfileWeight extends Weight {
subQueryWeight.extractTerms(set);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return subQueryWeight.isCacheable(ctx);
}
}

View File

@ -77,6 +77,11 @@ public final class DocValuesSliceQuery extends SliceQuery {
return new ConstantScoreScorer(this, score(), twoPhase);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return DocValues.isCacheable(ctx, getField());
}
};
}
}

View File

@ -63,6 +63,11 @@ public final class TermsSliceQuery extends SliceQuery {
final DocIdSetIterator leafIt = disi.iterator();
return new ConstantScoreScorer(this, score(), leafIt);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return true;
}
};
}

View File

@ -103,11 +103,6 @@ public class MinScoreScorerTests extends LuceneTestCase {
final int idx = Arrays.binarySearch(docs, docID());
return scores[idx];
}
@Override
public int freq() throws IOException {
return 1;
}
};
}

View File

@ -50,11 +50,6 @@ public class ScriptQueryBuilderTests extends AbstractQueryTestCase<ScriptQueryBu
@Override
protected void doAssertLuceneQuery(ScriptQueryBuilder queryBuilder, Query query, SearchContext context) throws IOException {
assertThat(query, instanceOf(ScriptQueryBuilder.ScriptQuery.class));
// make sure the query would not get cached
ScriptQuery sQuery = (ScriptQuery) query;
ScriptQuery clone = new ScriptQuery(sQuery.script, sQuery.filterScript);
assertFalse(sQuery.equals(clone));
assertFalse(sQuery.hashCode() == clone.hashCode());
}
public void testIllegalConstructorArg() {

View File

@ -79,6 +79,11 @@ public class IndicesQueryCacheTests extends ESTestCase {
public Scorer scorer(LeafReaderContext context) throws IOException {
return new ConstantScoreScorer(this, score(), DocIdSetIterator.all(context.reader().maxDoc()));
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return true;
}
};
}
@ -364,6 +369,11 @@ public class IndicesQueryCacheTests extends ESTestCase {
return weight.scorerSupplier(context);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return true;
}
}
public void testDelegatesScorerSupplier() throws Exception {

View File

@ -237,7 +237,6 @@ public class QueryProfilerTests extends ESTestCase {
@Override
public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
final Weight weight = this;
return new ScorerSupplier() {
@Override
@ -251,6 +250,11 @@ public class QueryProfilerTests extends ESTestCase {
}
};
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return true;
}
};
}
}

View File

@ -1,7 +1,7 @@
:version: 7.0.0-alpha1
:major-version: 7.x
:lucene_version: 7.1.0
:lucene_version_path: 7_1_0
:lucene_version: 7.2.0-SNAPSHOT
:lucene_version_path: 7_2_0
:branch: master
:jdk: 1.8.0_131

View File

@ -1 +0,0 @@
714927eb1d1db641bff9aa658e7e112c368f3e6d

View File

@ -0,0 +1 @@
5d4b0551a5f745ddf630184b8f63d9d03b4f4003

View File

@ -23,6 +23,7 @@ import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.DoubleValues;
import org.apache.lucene.search.DoubleValuesSource;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import java.io.IOException;
@ -72,4 +73,14 @@ final class ReplaceableConstDoubleValueSource extends DoubleValuesSource {
public String toString() {
return getClass().getSimpleName();
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return false;
}
@Override
public DoubleValuesSource rewrite(IndexSearcher reader) throws IOException {
return this;
}
}

View File

@ -37,10 +37,6 @@ public class ScoreTests extends ScriptTestCase {
return 0;
}
@Override
public int freq() throws IOException {
throw new UnsupportedOperationException();
}
@Override
public DocIdSetIterator iterator() {
throw new UnsupportedOperationException();
}

View File

@ -19,6 +19,7 @@
package org.apache.lucene.queries;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.ConstantScoreScorer;
import org.apache.lucene.search.ConstantScoreWeight;
@ -107,6 +108,11 @@ public final class BinaryDocValuesRangeQuery extends Query {
};
return new ConstantScoreScorer(this, score(), iterator);
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return DocValues.isCacheable(ctx, fieldName);
}
};
}

View File

@ -164,6 +164,13 @@ final class PercolateQuery extends Query implements Accountable {
};
}
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
// This query uses a significant amount of memory, let's never
// cache it or compound queries that wrap it.
return false;
}
};
}
@ -261,11 +268,6 @@ final class PercolateQuery extends Query implements Accountable {
};
}
@Override
public final int freq() throws IOException {
return approximation.freq();
}
@Override
public final int docID() {
return approximation.docID();

View File

@ -695,6 +695,11 @@ public class CandidateQueryTests extends ESSingleNodeTestCase {
}
};
}
@Override
public boolean isCacheable(LeafReaderContext ctx) {
return false; // doesn't matter
}
};
}

View File

@ -1 +0,0 @@
d9a640081289c9c50da08479ff198b579df71c26

View File

@ -0,0 +1 @@
f18454bf4be4698f7e6f3a7c950f75ee3fa4436e

View File

@ -1 +0,0 @@
a2ca81efc31d857fa2ade104dcdb3fed20c95ea0

View File

@ -0,0 +1 @@
a09ec54a9434987a16a4d9b68ca301a5a3bf4123

View File

@ -1 +0,0 @@
42058220ada77c4c5340e8383f62a4398e10a8ce

View File

@ -0,0 +1 @@
f86560f04296d8d34fed535abbf8002e0985b1df

View File

@ -1 +0,0 @@
2769d7f7330c78aea1edf4d8cd2eb111564c6800

View File

@ -0,0 +1 @@
dec7d0bee4de2ac8eb5aec9cb5d244c5b405bc15

View File

@ -1 +0,0 @@
2bec616dc5bb33df9d0beddf6a9565ef14a227ff

View File

@ -0,0 +1 @@
8469b05416dcaf6ddeebcdabb551b4ccf009f7c2

View File

@ -1 +0,0 @@
0e78e3e59b7bdf6e1aa24ff8289cc1246248f642

View File

@ -0,0 +1 @@
6de67186607bde2ac66b216af4e1fadffaf43e21