diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 7ae02afb2fb..210c8187978 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -26,6 +26,8 @@ Improvements * LUCENE-9018: ConcatenateGraphFilter now has a configurable separator. (Stanislav Mikulchik, David Smiley) +* LUCENE-9036: ExitableDirectoryReader may interupt scaning over DocValues (Mikhail Khludnev) + Optimizations * LUCENE-8928: When building a kd-tree for dimensions n > 2, compute exact bounds for an inner node every N splits diff --git a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java index 03de3c66f23..f66a9017adc 100644 --- a/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java +++ b/lucene/core/src/java/org/apache/lucene/index/ExitableDirectoryReader.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.lucene.index.FilterLeafReader.FilterTerms; import org.apache.lucene.index.FilterLeafReader.FilterTermsEnum; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.automaton.CompiledAutomaton; @@ -69,7 +70,9 @@ public class ExitableDirectoryReader extends FilterDirectoryReader { */ public static class ExitableFilterAtomicReader extends FilterLeafReader { - private QueryTimeout queryTimeout; + final private QueryTimeout queryTimeout; + + final static int DOCS_BETWEEN_TIMEOUT_CHECK = 1000; /** Constructor **/ public ExitableFilterAtomicReader(LeafReader in, QueryTimeout queryTimeout) { @@ -107,6 +110,216 @@ public class ExitableDirectoryReader extends FilterDirectoryReader { return in.getCoreCacheHelper(); } + @Override + public NumericDocValues getNumericDocValues(String field) throws IOException { + final NumericDocValues numericDocValues = super.getNumericDocValues(field); + if (numericDocValues == null) { + return null; + } + return (queryTimeout.isTimeoutEnabled()) ? new FilterNumericDocValues(numericDocValues) { + private int docToCheck = 0; + @Override + public int advance(int target) throws IOException { + final int advance = super.advance(target); + if (advance >= docToCheck) { + checkAndThrow(in); + docToCheck = advance + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advance; + } + @Override + public boolean advanceExact(int target) throws IOException { + final boolean advanceExact = super.advanceExact(target); + if (target >= docToCheck) { + checkAndThrow(in); + docToCheck=target + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advanceExact; + } + @Override + public int nextDoc() throws IOException { + final int nextDoc = super.nextDoc(); + if (nextDoc >= docToCheck) { + checkAndThrow(in); + docToCheck = nextDoc + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return nextDoc; + } + }: numericDocValues; + } + + @Override + public BinaryDocValues getBinaryDocValues(String field) throws IOException { + final BinaryDocValues binaryDocValues = super.getBinaryDocValues(field); + if (binaryDocValues == null) { + return null; + } + return (queryTimeout.isTimeoutEnabled()) ? new FilterBinaryDocValues(binaryDocValues) { + private int docToCheck = 0; + @Override + public int advance(int target) throws IOException { + final int advance = super.advance(target); + if (target >= docToCheck) { + checkAndThrow(in); + docToCheck = target + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advance; + } + @Override + public boolean advanceExact(int target) throws IOException { + final boolean advanceExact = super.advanceExact(target); + if (target >= docToCheck) { + checkAndThrow(in); + docToCheck = target + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advanceExact; + } + @Override + public int nextDoc() throws IOException { + final int nextDoc = super.nextDoc(); + if (nextDoc >= docToCheck) { + checkAndThrow(in); + docToCheck = nextDoc + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return nextDoc; + } + }: binaryDocValues; + } + + @Override + public SortedDocValues getSortedDocValues(String field) throws IOException { + final SortedDocValues sortedDocValues = super.getSortedDocValues(field); + if (sortedDocValues == null) { + return null; + } + return (queryTimeout.isTimeoutEnabled()) ? new FilterSortedDocValues(sortedDocValues) { + + private int docToCheck = 0; + + @Override + public int advance(int target) throws IOException { + final int advance = super.advance(target); + if (advance >= docToCheck) { + checkAndThrow(in); + docToCheck = advance + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advance; + } + @Override + public boolean advanceExact(int target) throws IOException { + final boolean advanceExact = super.advanceExact(target); + if (target >= docToCheck) { + checkAndThrow(in); + docToCheck = target + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advanceExact; + } + @Override + public int nextDoc() throws IOException { + final int nextDoc = super.nextDoc(); + if (nextDoc >= docToCheck) { + checkAndThrow(in); + docToCheck = nextDoc + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return nextDoc; + } + }: sortedDocValues; + } + + @Override + public SortedNumericDocValues getSortedNumericDocValues(String field) throws IOException { + final SortedNumericDocValues sortedNumericDocValues = super.getSortedNumericDocValues(field); + if (sortedNumericDocValues == null) { + return null; + } + return (queryTimeout.isTimeoutEnabled()) ? new FilterSortedNumericDocValues(sortedNumericDocValues) { + + private int docToCheck = 0; + + @Override + public int advance(int target) throws IOException { + final int advance = super.advance(target); + if (advance >= docToCheck) { + checkAndThrow(in); + docToCheck = advance + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advance; + } + @Override + public boolean advanceExact(int target) throws IOException { + final boolean advanceExact = super.advanceExact(target); + if (target >= docToCheck) { + checkAndThrow(in); + docToCheck = target + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advanceExact; + } + @Override + public int nextDoc() throws IOException { + final int nextDoc = super.nextDoc(); + if (nextDoc >= docToCheck) { + checkAndThrow(in); + docToCheck = nextDoc + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return nextDoc; + } + }: sortedNumericDocValues; + } + + @Override + public SortedSetDocValues getSortedSetDocValues(String field) throws IOException { + final SortedSetDocValues sortedSetDocValues = super.getSortedSetDocValues(field); + if (sortedSetDocValues == null) { + return null; + } + return (queryTimeout.isTimeoutEnabled()) ? new FilterSortedSetDocValues(sortedSetDocValues) { + + private int docToCheck=0; + + @Override + public int advance(int target) throws IOException { + final int advance = super.advance(target); + if (advance >= docToCheck) { + checkAndThrow(in); + docToCheck = advance + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advance; + } + @Override + public boolean advanceExact(int target) throws IOException { + final boolean advanceExact = super.advanceExact(target); + if (target >= docToCheck) { + checkAndThrow(in); + docToCheck = target + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return advanceExact; + } + @Override + public int nextDoc() throws IOException { + final int nextDoc = super.nextDoc(); + if (nextDoc >= docToCheck) { + checkAndThrow(in); + docToCheck = nextDoc + DOCS_BETWEEN_TIMEOUT_CHECK; + } + return nextDoc; + } + }: sortedSetDocValues; + } + + /** + * Throws {@link ExitingReaderException} if {@link QueryTimeout#shouldExit()} returns true, + * or if {@link Thread#interrupted()} returns true. + * @param in underneath docValues + */ + private void checkAndThrow(DocIdSetIterator in) { + if (queryTimeout.shouldExit()) { + throw new ExitingReaderException("The request took too long to iterate over doc values. Timeout: " + + queryTimeout.toString() + ", DocValues=" + in + ); + } else if (Thread.interrupted()) { + throw new ExitingReaderException("Interrupted while iterating over point values. PointValues=" + in); + } + } } /** diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java b/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java index 650ad0406f5..925f65e62c2 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilterBinaryDocValues.java @@ -18,6 +18,7 @@ package org.apache.lucene.index; import java.io.IOException; +import java.util.Objects; import org.apache.lucene.util.BytesRef; @@ -31,6 +32,7 @@ public abstract class FilterBinaryDocValues extends BinaryDocValues { /** Sole constructor */ protected FilterBinaryDocValues(BinaryDocValues in) { + Objects.requireNonNull(in); this.in = in; } diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java index bd00cf2b030..81a820afc68 100644 --- a/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java +++ b/lucene/core/src/java/org/apache/lucene/index/FilterNumericDocValues.java @@ -18,6 +18,7 @@ package org.apache.lucene.index; import java.io.IOException; +import java.util.Objects; /** * Delegates all methods to a wrapped {@link NumericDocValues}. @@ -29,6 +30,7 @@ public abstract class FilterNumericDocValues extends NumericDocValues { /** Sole constructor */ protected FilterNumericDocValues(NumericDocValues in) { + Objects.requireNonNull(in); this.in = in; } diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterSortedDocValues.java b/lucene/core/src/java/org/apache/lucene/index/FilterSortedDocValues.java new file mode 100644 index 00000000000..e329c29983d --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/FilterSortedDocValues.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Objects; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.CompiledAutomaton; + +/** + * Delegates all methods to a wrapped {@link SortedDocValues}. + */ +public abstract class FilterSortedDocValues extends SortedDocValues { + + /** Wrapped values */ + protected final SortedDocValues in; + + /** Sole constructor */ + public FilterSortedDocValues(SortedDocValues in) { + Objects.requireNonNull(in); + this.in = in; + } + + @Override + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); + } + + @Override + public int ordValue() throws IOException { + return in.ordValue(); + } + + @Override + public BytesRef lookupOrd(int ord) throws IOException { + return in.lookupOrd(ord); + } + + @Override + public BytesRef binaryValue() throws IOException { + return in.binaryValue(); + } + + @Override + public int getValueCount() { + return in.getValueCount(); + } + + @Override + public int lookupTerm(BytesRef key) throws IOException { + return in.lookupTerm(key); + } + + @Override + public TermsEnum termsEnum() throws IOException { + return in.termsEnum(); + } + + @Override + public TermsEnum intersect(CompiledAutomaton automaton) throws IOException { + return in.intersect(automaton); + } + + @Override + public int docID() { + return in.docID(); + } + + @Override + public int nextDoc() throws IOException { + return in.nextDoc(); + } + + @Override + public int advance(int target) throws IOException { + return in.advance(target); + } + + @Override + public long cost() { + return in.cost(); + } +} diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterSortedNumericDocValues.java b/lucene/core/src/java/org/apache/lucene/index/FilterSortedNumericDocValues.java new file mode 100644 index 00000000000..1597eb02e22 --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/FilterSortedNumericDocValues.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Objects; + +/** + * Delegates all methods to a wrapped {@link SortedNumericDocValues}. + */ +public abstract class FilterSortedNumericDocValues extends SortedNumericDocValues { + + /** Wrapped values */ + protected final SortedNumericDocValues in; + + /** Sole constructor */ + public FilterSortedNumericDocValues(SortedNumericDocValues in) { + Objects.requireNonNull(in); + this.in = in; + } + + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); + } + + public long nextValue() throws IOException { + return in.nextValue(); + } + + public int docValueCount() { + return in.docValueCount(); + } + + public int docID() { + return in.docID(); + } + + public int nextDoc() throws IOException { + return in.nextDoc(); + } + + public int advance(int target) throws IOException { + return in.advance(target); + } + + public long cost() { + return in.cost(); + } + + + +} diff --git a/lucene/core/src/java/org/apache/lucene/index/FilterSortedSetDocValues.java b/lucene/core/src/java/org/apache/lucene/index/FilterSortedSetDocValues.java new file mode 100644 index 00000000000..b403460e7ee --- /dev/null +++ b/lucene/core/src/java/org/apache/lucene/index/FilterSortedSetDocValues.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.lucene.index; + +import java.io.IOException; +import java.util.Objects; + +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.CompiledAutomaton; + +/** + * Delegates all methods to a wrapped {@link SortedSetDocValues}. + */ +public class FilterSortedSetDocValues extends SortedSetDocValues { + + /** Wrapped values */ + protected final SortedSetDocValues in; + + /** Initializes delegate */ + public FilterSortedSetDocValues(SortedSetDocValues in) { + Objects.requireNonNull(in); + this.in = in; + } + + public boolean advanceExact(int target) throws IOException { + return in.advanceExact(target); + } + + public long nextOrd() throws IOException { + return in.nextOrd(); + } + + public BytesRef lookupOrd(long ord) throws IOException { + return in.lookupOrd(ord); + } + + public long getValueCount() { + return in.getValueCount(); + } + + public long lookupTerm(BytesRef key) throws IOException { + return in.lookupTerm(key); + } + + public TermsEnum termsEnum() throws IOException { + return in.termsEnum(); + } + + public TermsEnum intersect(CompiledAutomaton automaton) throws IOException { + return in.intersect(automaton); + } + + public int docID() { + return in.docID(); + } + + public int nextDoc() throws IOException { + return in.nextDoc(); + } + + public int advance(int target) throws IOException { + return in.advance(target); + } + + public long cost() { + return in.cost(); + } + + +} diff --git a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java index 6ca9800c973..e1e45f0bc81 100644 --- a/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java +++ b/lucene/core/src/test/org/apache/lucene/index/TestExitableDirectoryReader.java @@ -17,12 +17,19 @@ package org.apache.lucene.index; import java.io.IOException; +import java.util.Arrays; import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.BinaryDocValuesField; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.SortedSetDocValuesField; import org.apache.lucene.index.ExitableDirectoryReader.ExitingReaderException; +import org.apache.lucene.search.DocIdSetIterator; import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; @@ -275,5 +282,99 @@ public class TestExitableDirectoryReader extends LuceneTestCase { } }; } + + @FunctionalInterface + interface DvFactory { + DocValuesIterator create(LeafReader leaf) throws IOException; + } + + public void testDocValues() throws IOException { + Directory directory = newDirectory(); + IndexWriter writer = new IndexWriter(directory, newIndexWriterConfig(new MockAnalyzer(random()))); + + Document d1 = new Document(); + addDVs(d1, 10); + writer.addDocument(d1); + + Document d2 = new Document(); + addDVs(d2, 100); + writer.addDocument(d2); + + Document d3 = new Document(); + addDVs(d3, 1000); + writer.addDocument(d3); + + writer.forceMerge(1); + writer.commit(); + writer.close(); + + DirectoryReader directoryReader; + DirectoryReader exitableDirectoryReader; + + for (DvFactory dvFactory : Arrays.asList( + (r) -> r.getSortedDocValues("sorted"), + (r) -> r.getSortedSetDocValues("sortedset"), + (r) -> r.getSortedNumericDocValues("sortednumeric"), + (r) -> r.getNumericDocValues("numeric"), + (r) -> r.getBinaryDocValues("binary") + )) + { + directoryReader = DirectoryReader.open(directory); + exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, immediateQueryTimeout()); + + { + IndexReader reader = new TestReader(getOnlyLeafReader(exitableDirectoryReader)); + + expectThrows(ExitingReaderException.class, () -> { + LeafReader leaf = reader.leaves().get(0).reader(); + DocValuesIterator iter = dvFactory.create(leaf); + scan(leaf, iter); + }); + reader.close(); + } + + directoryReader = DirectoryReader.open(directory); + exitableDirectoryReader = new ExitableDirectoryReader(directoryReader, random().nextBoolean()? + infiniteQueryTimeout() : disabledQueryTimeout()); + { + IndexReader reader = new TestReader(getOnlyLeafReader(exitableDirectoryReader)); + final LeafReader leaf = reader.leaves().get(0).reader(); + scan(leaf, dvFactory.create(leaf)); + assertNull(leaf.getNumericDocValues("absent")); + assertNull(leaf.getBinaryDocValues("absent")); + assertNull(leaf.getSortedDocValues("absent")); + assertNull(leaf.getSortedNumericDocValues("absent")); + assertNull(leaf.getSortedSetDocValues("absent")); + + reader.close(); + } + } + + directory.close(); + + } + + static private void scan(LeafReader leaf, DocValuesIterator iter ) throws IOException { + for (iter.nextDoc(); iter.docID()!=DocIdSetIterator.NO_MORE_DOCS + && iter.docID() + + + + diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java b/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java index 7cea5f0a057..3c757ad7bf9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java @@ -125,19 +125,32 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase { counter = 1; UpdateRequest req = new UpdateRequest(); - for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ ) - req.add(sdoc("id", Integer.toString(counter), "name", "a" + counter, + for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ ) { + final String v = "a" + counter; + req.add(sdoc("id", Integer.toString(counter), "name", v, + "name_dv", v, + "name_dvs", v,"name_dvs", v+"1", "num",""+counter)); + } counter++; - for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ ) - req.add(sdoc("id", Integer.toString(counter), "name", "b" + counter, + for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ ) { + final String v = "b" + counter; + req.add(sdoc("id", Integer.toString(counter), "name", v, + "name_dv", v, + "name_dvs", v,"name_dvs", v+"1", "num",""+counter)); + } counter++; - for(; counter % NUM_DOCS_PER_TYPE != 0; counter++ ) - req.add(sdoc("id", Integer.toString(counter), "name", "dummy term doc" + counter, + for(; counter % NUM_DOCS_PER_TYPE != 0; counter++ ) { + final String v = "dummy term doc" + counter; + req.add(sdoc("id", Integer.toString(counter), "name", + v, + "name_dv", v, + "name_dvs", v,"name_dvs", v+"1", "num",""+counter)); + } req.commit(client, COLLECTION); } @@ -226,7 +239,10 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase { SolrParams cases[] = new SolrParams[] { params( "sort","query($q,1) asc"), params("rows","0", "facet","true", "facet.method", "enum", "facet.field", "name"), - params("rows","0", "json.facet","{ ids: { type: range, field : num, start : 1, end : 99, gap : 9 }}") + params("rows","0", "json.facet","{ ids: { type: range, field : num, start : 1, end : 99, gap : 9 }}"), + params("q", "*:*", "rows","0", "json.facet","{ ids: { type: field, field : num}}"), + params("q", "*:*", "rows","0", "json.facet","{ ids: { type: field, field : name_dv}}"), + params("q", "*:*", "rows","0", "json.facet","{ ids: { type: field, field : name_dvs}}") }; // add more cases here params.add(cases[random().nextInt(cases.length)]);