mirror of https://github.com/apache/lucene.git
SOLR-12147: Don't use memory-intensive codecs to test uninvert limits
This separates out TestDocTermOrds.testTriggerUninvertLimit into its own test class, and suppresses Memory codec on it
This commit is contained in:
parent
c70cceaee5
commit
37e065007f
|
@ -21,7 +21,6 @@ import java.util.ArrayList;
|
|||
import java.util.Arrays;
|
||||
import java.util.HashSet;
|
||||
import java.util.List;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
|
@ -137,62 +136,6 @@ public class TestDocTermOrds extends LuceneTestCase {
|
|||
dir.close();
|
||||
}
|
||||
|
||||
/* UnInvertedField had a reference block limitation of 2^24. This unit test triggered it.
|
||||
*
|
||||
* With the current code, the test verifies that the old limit no longer applies.
|
||||
* New limit is 2^31, which is not very realistic to unit-test. */
|
||||
@SuppressWarnings({"ConstantConditions", "PointlessBooleanExpression"})
|
||||
@Nightly
|
||||
public void testTriggerUnInvertLimit() throws IOException {
|
||||
final boolean SHOULD_TRIGGER = false; // Set this to true to use the test with the old implementation
|
||||
|
||||
assumeFalse("Don't run this massive test with MemoryPostingsFormat, as it can OOM",
|
||||
Objects.equals(Codec.getDefault().postingsFormat().getName(), "Memory"));
|
||||
|
||||
// Ensure enough terms inside of a single UnInvert-pass-structure to trigger the limit
|
||||
final int REF_LIMIT = (int) Math.pow(2, 24); // Maximum number of references within a single pass-structure
|
||||
final int DOCS = (1<<16)-1; // The number of documents within a single pass (simplified)
|
||||
final int TERMS = REF_LIMIT/DOCS; // Each document must have this many references aka terms hit limit
|
||||
|
||||
Directory dir = newDirectory();
|
||||
final RandomIndexWriter w = new RandomIndexWriter(random(), dir,
|
||||
newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
|
||||
Document doc = new Document();
|
||||
Field field = newTextField("field", "", Field.Store.NO);
|
||||
doc.add(field);
|
||||
|
||||
StringBuilder sb = new StringBuilder(TERMS*(Integer.toString(TERMS).length()+1));
|
||||
for (int i = 0 ; i < TERMS ; i++) {
|
||||
sb.append(" ").append(Integer.toString(i));
|
||||
}
|
||||
field.setStringValue(sb.toString());
|
||||
|
||||
for (int i = 0 ; i < DOCS ; i++) {
|
||||
w.addDocument(doc);
|
||||
}
|
||||
//System.out.println("\n Finished adding " + DOCS + " documents of " + TERMS + " unique terms");
|
||||
final IndexReader r = w.getReader();
|
||||
w.close();
|
||||
|
||||
try {
|
||||
final LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
|
||||
TestUtil.checkReader(ar);
|
||||
final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field"); // bigTerms turned off
|
||||
if (SHOULD_TRIGGER) {
|
||||
fail("DocTermOrds should have failed with a \"Too many values for UnInvertedField\" message");
|
||||
}
|
||||
} catch (IllegalStateException e) {
|
||||
if (!SHOULD_TRIGGER) {
|
||||
fail("DocTermsOrd should not have failed with this implementation, but got exception " +
|
||||
e.getClass().getSimpleName() + " with message " + e.getMessage());
|
||||
}
|
||||
// This is (hopefully) "Too many values for UnInvertedField faceting on field field", so all is as expected
|
||||
} finally {
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
}
|
||||
|
||||
public void testRandom() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
|
||||
|
|
|
@ -0,0 +1,88 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.solr.uninverting;
|
||||
|
||||
import java.io.IOException;
|
||||
|
||||
import org.apache.lucene.analysis.MockAnalyzer;
|
||||
import org.apache.lucene.document.Document;
|
||||
import org.apache.lucene.document.Field;
|
||||
import org.apache.lucene.index.IndexReader;
|
||||
import org.apache.lucene.index.LeafReader;
|
||||
import org.apache.lucene.index.RandomIndexWriter;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
import org.apache.solr.index.SlowCompositeReaderWrapper;
|
||||
|
||||
@LuceneTestCase.SuppressCodecs({"Memory"})
|
||||
public class TestDocTermOrdsUninvertLimit extends LuceneTestCase {
|
||||
|
||||
/* UnInvertedField had a reference block limitation of 2^24. This unit test triggered it.
|
||||
*
|
||||
* With the current code, the test verifies that the old limit no longer applies.
|
||||
* New limit is 2^31, which is not very realistic to unit-test. */
|
||||
@SuppressWarnings({"ConstantConditions", "PointlessBooleanExpression"})
|
||||
@Nightly
|
||||
public void testTriggerUnInvertLimit() throws IOException {
|
||||
final boolean SHOULD_TRIGGER = false; // Set this to true to use the test with the old implementation
|
||||
|
||||
// Ensure enough terms inside of a single UnInvert-pass-structure to trigger the limit
|
||||
final int REF_LIMIT = (int) Math.pow(2, 24); // Maximum number of references within a single pass-structure
|
||||
final int DOCS = (1<<16)-1; // The number of documents within a single pass (simplified)
|
||||
final int TERMS = REF_LIMIT/DOCS; // Each document must have this many references aka terms hit limit
|
||||
|
||||
Directory dir = newDirectory();
|
||||
final RandomIndexWriter w = new RandomIndexWriter(random(), dir,
|
||||
newIndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(newLogMergePolicy()));
|
||||
Document doc = new Document();
|
||||
Field field = newTextField("field", "", Field.Store.NO);
|
||||
doc.add(field);
|
||||
|
||||
StringBuilder sb = new StringBuilder(TERMS*(Integer.toString(TERMS).length()+1));
|
||||
for (int i = 0 ; i < TERMS ; i++) {
|
||||
sb.append(" ").append(Integer.toString(i));
|
||||
}
|
||||
field.setStringValue(sb.toString());
|
||||
|
||||
for (int i = 0 ; i < DOCS ; i++) {
|
||||
w.addDocument(doc);
|
||||
}
|
||||
//System.out.println("\n Finished adding " + DOCS + " documents of " + TERMS + " unique terms");
|
||||
final IndexReader r = w.getReader();
|
||||
w.close();
|
||||
|
||||
try {
|
||||
final LeafReader ar = SlowCompositeReaderWrapper.wrap(r);
|
||||
TestUtil.checkReader(ar);
|
||||
final DocTermOrds dto = new DocTermOrds(ar, ar.getLiveDocs(), "field"); // bigTerms turned off
|
||||
if (SHOULD_TRIGGER) {
|
||||
fail("DocTermOrds should have failed with a \"Too many values for UnInvertedField\" message");
|
||||
}
|
||||
} catch (IllegalStateException e) {
|
||||
if (!SHOULD_TRIGGER) {
|
||||
fail("DocTermsOrd should not have failed with this implementation, but got exception " +
|
||||
e.getClass().getSimpleName() + " with message " + e.getMessage());
|
||||
}
|
||||
// This is (hopefully) "Too many values for UnInvertedField faceting on field field", so all is as expected
|
||||
} finally {
|
||||
r.close();
|
||||
dir.close();
|
||||
}
|
||||
}
|
||||
}
|
Loading…
Reference in New Issue