LUCENE-10134: clean up the test from leaking threads and resources if an error occurs somewhere - this obscures the original cause of the problem.

This commit is contained in:
Dawid Weiss 2021-09-30 09:11:45 +02:00
parent 88b264a368
commit d2b88b7a0b
1 changed files with 438 additions and 446 deletions

View File

@ -45,7 +45,6 @@ import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory; import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Accountable; import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.NamedThreadFactory; import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.TestUtil; import org.apache.lucene.util.TestUtil;
@ -56,165 +55,162 @@ public class TestSortedSetDocValuesFacets extends FacetTestCase {
// randomly uses SortedSetDV // randomly uses SortedSetDV
public void testBasic() throws Exception { public void testBasic() throws Exception {
Directory dir = newDirectory();
FacetsConfig config = new FacetsConfig(); FacetsConfig config = new FacetsConfig();
config.setMultiValued("a", true); config.setMultiValued("a", true);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); try (Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
doc.add(new SortedSetDocValuesFacetField("a", "zoo"));
doc.add(new SortedSetDocValuesFacetField("b", "baz"));
writer.addDocument(config.build(doc));
if (random().nextBoolean()) {
writer.commit();
}
Document doc = new Document(); doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo")); doc.add(new SortedSetDocValuesFacetField("a", "foo"));
doc.add(new SortedSetDocValuesFacetField("a", "bar")); writer.addDocument(config.build(doc));
doc.add(new SortedSetDocValuesFacetField("a", "zoo"));
doc.add(new SortedSetDocValuesFacetField("b", "baz")); // NRT open
writer.addDocument(config.build(doc)); try (IndexReader r = writer.getReader()) {
if (random().nextBoolean()) { IndexSearcher searcher = newSearcher(r);
writer.commit();
// Per-top-reader state:
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull();
try {
Facets facets = getAllFacets(searcher, state, exec);
assertEquals(
"dim=a path=[] value=4 childCount=3\n foo (2)\n bar (1)\n zoo (1)\n",
facets.getTopChildren(10, "a").toString());
assertEquals(
"dim=b path=[] value=1 childCount=1\n baz (1)\n",
facets.getTopChildren(10, "b").toString());
// DrillDown:
DrillDownQuery q = new DrillDownQuery(config);
q.add("a", "foo");
q.add("b", "baz");
TopDocs hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
} finally {
if (exec != null) exec.shutdownNow();
}
}
} }
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
writer.addDocument(config.build(doc));
// NRT open
IndexSearcher searcher = newSearcher(writer.getReader());
// Per-top-reader state:
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull();
Facets facets = getAllFacets(searcher, state, exec);
assertEquals(
"dim=a path=[] value=4 childCount=3\n foo (2)\n bar (1)\n zoo (1)\n",
facets.getTopChildren(10, "a").toString());
assertEquals(
"dim=b path=[] value=1 childCount=1\n baz (1)\n",
facets.getTopChildren(10, "b").toString());
// DrillDown:
DrillDownQuery q = new DrillDownQuery(config);
q.add("a", "foo");
q.add("b", "baz");
TopDocs hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
if (exec != null) {
exec.shutdownNow();
}
writer.close();
IOUtils.close(searcher.getIndexReader(), dir);
} }
// See: LUCENE-10070 // See: LUCENE-10070
public void testCountAll() throws Exception { public void testCountAll() throws Exception {
Directory dir = newDirectory(); try (Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
FacetsConfig config = new FacetsConfig();
FacetsConfig config = new FacetsConfig(); Document doc = new Document();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); doc.add(new StringField("id", "0", Field.Store.NO));
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
writer.addDocument(config.build(doc));
Document doc = new Document(); doc = new Document();
doc.add(new StringField("id", "0", Field.Store.NO)); doc.add(new StringField("id", "1", Field.Store.NO));
doc.add(new SortedSetDocValuesFacetField("a", "foo")); doc.add(new SortedSetDocValuesFacetField("a", "bar"));
writer.addDocument(config.build(doc)); writer.addDocument(config.build(doc));
doc = new Document(); writer.deleteDocuments(new Term("id", "0"));
doc.add(new StringField("id", "1", Field.Store.NO));
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
writer.addDocument(config.build(doc));
writer.deleteDocuments(new Term("id", "0")); // NRT open
try (IndexReader r = writer.getReader()) {
IndexSearcher searcher = newSearcher(r);
// NRT open // Per-top-reader state:
IndexSearcher searcher = newSearcher(writer.getReader()); SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
// Per-top-reader state: Facets facets = new SortedSetDocValuesFacetCounts(state);
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
Facets facets = new SortedSetDocValuesFacetCounts(state); assertEquals(
"dim=a path=[] value=1 childCount=1\n bar (1)\n",
facets.getTopChildren(10, "a").toString());
assertEquals( ExecutorService exec =
"dim=a path=[] value=1 childCount=1\n bar (1)\n", new ThreadPoolExecutor(
facets.getTopChildren(10, "a").toString()); 1,
TestUtil.nextInt(random(), 2, 6),
Long.MAX_VALUE,
TimeUnit.MILLISECONDS,
new LinkedBlockingQueue<Runnable>(),
new NamedThreadFactory("TestIndexSearcher"));
try {
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, exec);
ExecutorService exec = assertEquals(
new ThreadPoolExecutor( "dim=a path=[] value=1 childCount=1\n bar (1)\n",
1, facets.getTopChildren(10, "a").toString());
TestUtil.nextInt(random(), 2, 6), } finally {
Long.MAX_VALUE, exec.shutdownNow();
TimeUnit.MILLISECONDS, }
new LinkedBlockingQueue<Runnable>(), }
new NamedThreadFactory("TestIndexSearcher")); }
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, exec);
assertEquals(
"dim=a path=[] value=1 childCount=1\n bar (1)\n",
facets.getTopChildren(10, "a").toString());
writer.close();
IOUtils.close(searcher.getIndexReader(), dir);
exec.shutdownNow();
} }
public void testBasicSingleValued() throws Exception { public void testBasicSingleValued() throws Exception {
Directory dir = newDirectory();
FacetsConfig config = new FacetsConfig(); FacetsConfig config = new FacetsConfig();
config.setMultiValued("a", false); config.setMultiValued("a", false);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); try (Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
doc.add(new SortedSetDocValuesFacetField("b", "bar"));
writer.addDocument(config.build(doc));
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
writer.addDocument(config.build(doc));
if (random().nextBoolean()) {
writer.commit();
}
Document doc = new Document(); doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo")); doc.add(new SortedSetDocValuesFacetField("a", "baz"));
doc.add(new SortedSetDocValuesFacetField("b", "bar")); writer.addDocument(config.build(doc));
writer.addDocument(config.build(doc));
doc = new Document(); // NRT open
doc.add(new SortedSetDocValuesFacetField("a", "foo")); try (IndexReader r = writer.getReader()) {
writer.addDocument(config.build(doc)); IndexSearcher searcher = newSearcher(r);
if (random().nextBoolean()) {
writer.commit(); // Per-top-reader state:
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull();
try {
Facets facets = getAllFacets(searcher, state, exec);
assertEquals(
"dim=a path=[] value=3 childCount=2\n foo (2)\n baz (1)\n",
facets.getTopChildren(10, "a").toString());
assertEquals(
"dim=b path=[] value=1 childCount=1\n bar (1)\n",
facets.getTopChildren(10, "b").toString());
// DrillDown:
DrillDownQuery q = new DrillDownQuery(config);
q.add("a", "foo");
q.add("b", "bar");
TopDocs hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
} finally {
if (exec != null) exec.shutdownNow();
}
}
} }
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "baz"));
writer.addDocument(config.build(doc));
// NRT open
IndexSearcher searcher = newSearcher(writer.getReader());
// Per-top-reader state:
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull();
Facets facets = getAllFacets(searcher, state, exec);
assertEquals(
"dim=a path=[] value=3 childCount=2\n foo (2)\n baz (1)\n",
facets.getTopChildren(10, "a").toString());
assertEquals(
"dim=b path=[] value=1 childCount=1\n bar (1)\n",
facets.getTopChildren(10, "b").toString());
// DrillDown:
DrillDownQuery q = new DrillDownQuery(config);
q.add("a", "foo");
q.add("b", "bar");
TopDocs hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
if (exec != null) {
exec.shutdownNow();
}
writer.close();
IOUtils.close(searcher.getIndexReader(), dir);
} }
public void testDrillDownOptions() throws Exception { public void testDrillDownOptions() throws Exception {
Directory dir = newDirectory();
FacetsConfig config = new FacetsConfig(); FacetsConfig config = new FacetsConfig();
config.setDrillDownTermsIndexing("c", FacetsConfig.DrillDownTermsIndexing.NONE); config.setDrillDownTermsIndexing("c", FacetsConfig.DrillDownTermsIndexing.NONE);
config.setDrillDownTermsIndexing( config.setDrillDownTermsIndexing(
@ -222,381 +218,377 @@ public class TestSortedSetDocValuesFacets extends FacetTestCase {
config.setDrillDownTermsIndexing("e", FacetsConfig.DrillDownTermsIndexing.ALL_PATHS_NO_DIM); config.setDrillDownTermsIndexing("e", FacetsConfig.DrillDownTermsIndexing.ALL_PATHS_NO_DIM);
config.setDrillDownTermsIndexing("f", FacetsConfig.DrillDownTermsIndexing.FULL_PATH_ONLY); config.setDrillDownTermsIndexing("f", FacetsConfig.DrillDownTermsIndexing.FULL_PATH_ONLY);
config.setDrillDownTermsIndexing("g", FacetsConfig.DrillDownTermsIndexing.ALL); config.setDrillDownTermsIndexing("g", FacetsConfig.DrillDownTermsIndexing.ALL);
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); try (Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
Document doc = new Document(); Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("c", "foo")); doc.add(new SortedSetDocValuesFacetField("c", "foo"));
doc.add(new SortedSetDocValuesFacetField("d", "foo")); doc.add(new SortedSetDocValuesFacetField("d", "foo"));
doc.add(new SortedSetDocValuesFacetField("e", "foo")); doc.add(new SortedSetDocValuesFacetField("e", "foo"));
doc.add(new SortedSetDocValuesFacetField("f", "foo")); doc.add(new SortedSetDocValuesFacetField("f", "foo"));
doc.add(new SortedSetDocValuesFacetField("g", "foo")); doc.add(new SortedSetDocValuesFacetField("g", "foo"));
writer.addDocument(config.build(doc)); writer.addDocument(config.build(doc));
if (random().nextBoolean()) { if (random().nextBoolean()) {
writer.commit(); writer.commit();
}
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
writer.addDocument(config.build(doc));
// NRT open
try (IndexReader r = writer.getReader()) {
IndexSearcher searcher = newSearcher(r);
// Drill down with different indexing configuration options
DrillDownQuery q = new DrillDownQuery(config);
q.add("c");
TopDocs hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("c", "foo");
hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("d");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("d", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("e");
hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("e", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("f");
hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("f", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("g");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("g", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
}
} }
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
writer.addDocument(config.build(doc));
// NRT open
IndexSearcher searcher = newSearcher(writer.getReader());
ExecutorService exec = randomExecutorServiceOrNull();
// Drill down with different indexing configuration options
DrillDownQuery q = new DrillDownQuery(config);
q.add("c");
TopDocs hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("c", "foo");
hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("d");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("d", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("e");
hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("e", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("f");
hits = searcher.search(q, 1);
assertEquals(0, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("f", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("g");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
q = new DrillDownQuery(config);
q.add("g", "foo");
hits = searcher.search(q, 1);
assertEquals(1, hits.totalHits.value);
if (exec != null) {
exec.shutdownNow();
}
writer.close();
IOUtils.close(searcher.getIndexReader(), dir);
} }
// LUCENE-5090 // LUCENE-5090
@SuppressWarnings("unused") @SuppressWarnings("unused")
public void testStaleState() throws Exception { public void testStaleState() throws Exception {
Directory dir = newDirectory(); try (Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); FacetsConfig config = new FacetsConfig();
FacetsConfig config = new FacetsConfig(); Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
writer.addDocument(config.build(doc));
Document doc = new Document(); try (IndexReader r = writer.getReader()) {
doc.add(new SortedSetDocValuesFacetField("a", "foo")); SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(r);
writer.addDocument(config.build(doc));
IndexReader r = writer.getReader(); doc = new Document();
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(r); doc.add(new SortedSetDocValuesFacetField("a", "bar"));
writer.addDocument(config.build(doc));
doc = new Document(); doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "bar")); doc.add(new SortedSetDocValuesFacetField("a", "baz"));
writer.addDocument(config.build(doc)); writer.addDocument(config.build(doc));
doc = new Document(); try (IndexReader r2 = writer.getReader()) {
doc.add(new SortedSetDocValuesFacetField("a", "baz")); IndexSearcher searcher = newSearcher(r2);
writer.addDocument(config.build(doc));
IndexSearcher searcher = newSearcher(writer.getReader()); FacetsCollector c = new FacetsCollector();
FacetsCollector c = new FacetsCollector(); searcher.search(new MatchAllDocsQuery(), c);
searcher.search(new MatchAllDocsQuery(), c); expectThrows(
IllegalStateException.class,
expectThrows( () -> {
IllegalStateException.class, new SortedSetDocValuesFacetCounts(state, c);
() -> { });
new SortedSetDocValuesFacetCounts(state, c); }
}); }
}
r.close();
writer.close();
searcher.getIndexReader().close();
dir.close();
} }
// LUCENE-5333 // LUCENE-5333
public void testSparseFacets() throws Exception { public void testSparseFacets() throws Exception {
Directory dir = newDirectory(); try (Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); FacetsConfig config = new FacetsConfig();
FacetsConfig config = new FacetsConfig(); Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo1"));
writer.addDocument(config.build(doc));
Document doc = new Document(); if (random().nextBoolean()) {
doc.add(new SortedSetDocValuesFacetField("a", "foo1")); writer.commit();
writer.addDocument(config.build(doc)); }
if (random().nextBoolean()) { doc = new Document();
writer.commit(); doc.add(new SortedSetDocValuesFacetField("a", "foo2"));
doc.add(new SortedSetDocValuesFacetField("b", "bar1"));
writer.addDocument(config.build(doc));
if (random().nextBoolean()) {
writer.commit();
}
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo3"));
doc.add(new SortedSetDocValuesFacetField("b", "bar2"));
doc.add(new SortedSetDocValuesFacetField("c", "baz1"));
writer.addDocument(config.build(doc));
// NRT open
try (IndexReader r = writer.getReader()) {
IndexSearcher searcher = newSearcher(r);
// Per-top-reader state:
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull();
try {
Facets facets = getAllFacets(searcher, state, exec);
// Ask for top 10 labels for any dims that have counts:
List<FacetResult> results = facets.getAllDims(10);
assertEquals(3, results.size());
assertEquals(
"dim=a path=[] value=3 childCount=3\n foo1 (1)\n foo2 (1)\n foo3 (1)\n",
results.get(0).toString());
assertEquals(
"dim=b path=[] value=2 childCount=2\n bar1 (1)\n bar2 (1)\n",
results.get(1).toString());
assertEquals(
"dim=c path=[] value=1 childCount=1\n baz1 (1)\n", results.get(2).toString());
Collection<Accountable> resources = state.getChildResources();
assertTrue(state.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
if (searcher.getIndexReader().leaves().size() > 1) {
assertTrue(state.ramBytesUsed() > 0);
assertFalse(resources.isEmpty());
assertTrue(resources.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
} else {
assertEquals(0, state.ramBytesUsed());
assertTrue(resources.isEmpty());
}
} finally {
if (exec != null) exec.shutdownNow();
}
}
} }
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo2"));
doc.add(new SortedSetDocValuesFacetField("b", "bar1"));
writer.addDocument(config.build(doc));
if (random().nextBoolean()) {
writer.commit();
}
doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo3"));
doc.add(new SortedSetDocValuesFacetField("b", "bar2"));
doc.add(new SortedSetDocValuesFacetField("c", "baz1"));
writer.addDocument(config.build(doc));
// NRT open
IndexSearcher searcher = newSearcher(writer.getReader());
writer.close();
// Per-top-reader state:
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull();
Facets facets = getAllFacets(searcher, state, exec);
// Ask for top 10 labels for any dims that have counts:
List<FacetResult> results = facets.getAllDims(10);
assertEquals(3, results.size());
assertEquals(
"dim=a path=[] value=3 childCount=3\n foo1 (1)\n foo2 (1)\n foo3 (1)\n",
results.get(0).toString());
assertEquals(
"dim=b path=[] value=2 childCount=2\n bar1 (1)\n bar2 (1)\n", results.get(1).toString());
assertEquals("dim=c path=[] value=1 childCount=1\n baz1 (1)\n", results.get(2).toString());
Collection<Accountable> resources = state.getChildResources();
assertTrue(state.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
if (searcher.getIndexReader().leaves().size() > 1) {
assertTrue(state.ramBytesUsed() > 0);
assertFalse(resources.isEmpty());
assertTrue(resources.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
} else {
assertEquals(0, state.ramBytesUsed());
assertTrue(resources.isEmpty());
}
if (exec != null) {
exec.shutdownNow();
}
searcher.getIndexReader().close();
dir.close();
} }
public void testSomeSegmentsMissing() throws Exception { public void testSomeSegmentsMissing() throws Exception {
Directory dir = newDirectory(); try (Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); FacetsConfig config = new FacetsConfig();
FacetsConfig config = new FacetsConfig(); Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo1"));
writer.addDocument(config.build(doc));
writer.commit();
Document doc = new Document(); doc = new Document();
doc.add(new SortedSetDocValuesFacetField("a", "foo1")); writer.addDocument(config.build(doc));
writer.addDocument(config.build(doc)); writer.commit();
writer.commit();
doc = new Document(); doc = new Document();
writer.addDocument(config.build(doc)); doc.add(new SortedSetDocValuesFacetField("a", "foo2"));
writer.commit(); writer.addDocument(config.build(doc));
writer.commit();
doc = new Document(); // NRT open
doc.add(new SortedSetDocValuesFacetField("a", "foo2")); try (IndexReader r = writer.getReader()) {
writer.addDocument(config.build(doc)); IndexSearcher searcher = newSearcher(r);
writer.commit();
// NRT open // Per-top-reader state:
IndexSearcher searcher = newSearcher(writer.getReader()); SortedSetDocValuesReaderState state =
writer.close(); new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
// Per-top-reader state: ExecutorService exec = randomExecutorServiceOrNull();
SortedSetDocValuesReaderState state = try {
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader()); Facets facets = getAllFacets(searcher, state, exec);
ExecutorService exec = randomExecutorServiceOrNull(); // Ask for top 10 labels for any dims that have counts:
Facets facets = getAllFacets(searcher, state, exec); assertEquals(
"dim=a path=[] value=2 childCount=2\n foo1 (1)\n foo2 (1)\n",
// Ask for top 10 labels for any dims that have counts: facets.getTopChildren(10, "a").toString());
assertEquals( } finally {
"dim=a path=[] value=2 childCount=2\n foo1 (1)\n foo2 (1)\n", if (exec != null) exec.shutdownNow();
facets.getTopChildren(10, "a").toString()); }
}
if (exec != null) {
exec.shutdownNow();
} }
searcher.getIndexReader().close();
dir.close();
} }
public void testRandom() throws Exception { public void testRandom() throws Exception {
int fullIterations = LuceneTestCase.TEST_NIGHTLY ? 20 : 3; int fullIterations = LuceneTestCase.TEST_NIGHTLY ? 20 : 3;
for (int fullIter = 0; fullIter < fullIterations; fullIter++) { for (int fullIter = 0; fullIter < fullIterations; fullIter++) {
String[] tokens = getRandomTokens(10); String[] tokens = getRandomTokens(10);
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), indexDir); try (Directory indexDir = newDirectory();
FacetsConfig config = new FacetsConfig(); RandomIndexWriter w = new RandomIndexWriter(random(), indexDir)) {
int numDocs = atLeast(1000); FacetsConfig config = new FacetsConfig();
// Most of the time allow up to 7 dims per doc, but occasionally limit all docs to a single int numDocs = atLeast(1000);
// dim: // Most of the time allow up to 7 dims per doc, but occasionally limit all docs to a single
int numDims; // dim:
if (random().nextInt(10) < 8) { int numDims;
numDims = TestUtil.nextInt(random(), 1, 7); if (random().nextInt(10) < 8) {
} else { numDims = TestUtil.nextInt(random(), 1, 7);
numDims = 1;
}
List<TestDoc> testDocs = getRandomDocs(tokens, numDocs, numDims);
for (TestDoc testDoc : testDocs) {
Document doc = new Document();
doc.add(newStringField("content", testDoc.content, Field.Store.NO));
for (int j = 0; j < numDims; j++) {
if (testDoc.dims[j] != null) {
doc.add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j]));
}
}
w.addDocument(config.build(doc));
}
// NRT open
IndexSearcher searcher = newSearcher(w.getReader());
// Per-top-reader state:
SortedSetDocValuesReaderState state =
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull();
int iters = atLeast(100);
for (int iter = 0; iter < iters; iter++) {
String searchToken = tokens[random().nextInt(tokens.length)];
if (VERBOSE) {
System.out.println("\nTEST: iter content=" + searchToken);
}
FacetsCollector fc = new FacetsCollector();
FacetsCollector.search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
Facets facets;
if (exec != null) {
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, fc, exec);
} else { } else {
facets = new SortedSetDocValuesFacetCounts(state, fc); numDims = 1;
} }
List<TestDoc> testDocs = getRandomDocs(tokens, numDocs, numDims);
// Slow, yet hopefully bug-free, faceting: for (TestDoc testDoc : testDocs) {
@SuppressWarnings({"rawtypes", "unchecked"}) Document doc = new Document();
Map<String, Integer>[] expectedCounts = new HashMap[numDims]; doc.add(newStringField("content", testDoc.content, Field.Store.NO));
for (int i = 0; i < numDims; i++) { for (int j = 0; j < numDims; j++) {
expectedCounts[i] = new HashMap<>(); if (testDoc.dims[j] != null) {
} doc.add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j]));
for (TestDoc doc : testDocs) {
if (doc.content.equals(searchToken)) {
for (int j = 0; j < numDims; j++) {
if (doc.dims[j] != null) {
Integer v = expectedCounts[j].get(doc.dims[j]);
if (v == null) {
expectedCounts[j].put(doc.dims[j], 1);
} else {
expectedCounts[j].put(doc.dims[j], v.intValue() + 1);
}
}
} }
} }
w.addDocument(config.build(doc));
} }
List<FacetResult> expected = new ArrayList<>(); // NRT open
for (int i = 0; i < numDims; i++) { try (IndexReader r = w.getReader()) {
List<LabelAndValue> labelValues = new ArrayList<>(); IndexSearcher searcher = newSearcher(r);
int totCount = 0;
for (Map.Entry<String, Integer> ent : expectedCounts[i].entrySet()) { // Per-top-reader state:
labelValues.add(new LabelAndValue(ent.getKey(), ent.getValue())); SortedSetDocValuesReaderState state =
totCount += ent.getValue(); new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
} ExecutorService exec = randomExecutorServiceOrNull();
sortLabelValues(labelValues); try {
if (totCount > 0) { int iters = atLeast(100);
expected.add( for (int iter = 0; iter < iters; iter++) {
new FacetResult( String searchToken = tokens[random().nextInt(tokens.length)];
"dim" + i, if (VERBOSE) {
new String[0], System.out.println("\nTEST: iter content=" + searchToken);
totCount, }
labelValues.toArray(new LabelAndValue[labelValues.size()]), FacetsCollector fc = new FacetsCollector();
labelValues.size())); FacetsCollector.search(
searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
Facets facets;
if (exec != null) {
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, fc, exec);
} else {
facets = new SortedSetDocValuesFacetCounts(state, fc);
}
// Slow, yet hopefully bug-free, faceting:
@SuppressWarnings({"rawtypes", "unchecked"})
Map<String, Integer>[] expectedCounts = new HashMap[numDims];
for (int i = 0; i < numDims; i++) {
expectedCounts[i] = new HashMap<>();
}
for (TestDoc doc : testDocs) {
if (doc.content.equals(searchToken)) {
for (int j = 0; j < numDims; j++) {
if (doc.dims[j] != null) {
Integer v = expectedCounts[j].get(doc.dims[j]);
if (v == null) {
expectedCounts[j].put(doc.dims[j], 1);
} else {
expectedCounts[j].put(doc.dims[j], v.intValue() + 1);
}
}
}
}
}
List<FacetResult> expected = new ArrayList<>();
for (int i = 0; i < numDims; i++) {
List<LabelAndValue> labelValues = new ArrayList<>();
int totCount = 0;
for (Map.Entry<String, Integer> ent : expectedCounts[i].entrySet()) {
labelValues.add(new LabelAndValue(ent.getKey(), ent.getValue()));
totCount += ent.getValue();
}
sortLabelValues(labelValues);
if (totCount > 0) {
expected.add(
new FacetResult(
"dim" + i,
new String[0],
totCount,
labelValues.toArray(new LabelAndValue[labelValues.size()]),
labelValues.size()));
}
}
// Sort by highest value, tie break by value:
sortFacetResults(expected);
List<FacetResult> actual = facets.getAllDims(10);
// Messy: fixup ties
// sortTies(actual);
assertEquals(expected, actual);
}
} finally {
if (exec != null) exec.shutdownNow();
} }
} }
// Sort by highest value, tie break by value:
sortFacetResults(expected);
List<FacetResult> actual = facets.getAllDims(10);
// Messy: fixup ties
// sortTies(actual);
assertEquals(expected, actual);
} }
if (exec != null) {
exec.shutdownNow();
}
w.close();
IOUtils.close(searcher.getIndexReader(), indexDir, taxoDir);
} }
} }
public void testNonExistentDimension() throws Exception { public void testNonExistentDimension() throws Exception {
Directory dir = newDirectory(); try (Directory dir = newDirectory();
FacetsConfig config = new FacetsConfig(); RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
RandomIndexWriter writer = new RandomIndexWriter(random(), dir); FacetsConfig config = new FacetsConfig();
Document doc = new Document(); Document doc = new Document();
doc.add(new SortedSetDocValuesFacetField("foo", "bar")); doc.add(new SortedSetDocValuesFacetField("foo", "bar"));
writer.addDocument(config.build(doc)); writer.addDocument(config.build(doc));
writer.commit(); writer.commit();
IndexSearcher searcher = newSearcher(writer.getReader()); try (IndexReader r = writer.getReader()) {
SortedSetDocValuesReaderState state = IndexSearcher searcher = newSearcher(r);
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
ExecutorService exec = randomExecutorServiceOrNull(); SortedSetDocValuesReaderState state =
Facets facets = getAllFacets(searcher, state, exec); new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
FacetResult result = facets.getTopChildren(5, "non-existent dimension");
// make sure the result is null (and no exception was thrown) ExecutorService exec = randomExecutorServiceOrNull();
assertNull(result); try {
Facets facets = getAllFacets(searcher, state, exec);
FacetResult result = facets.getTopChildren(5, "non-existent dimension");
writer.close(); // make sure the result is null (and no exception was thrown)
IOUtils.close(searcher.getIndexReader(), dir); assertNull(result);
if (exec != null) { } finally {
exec.shutdownNow(); if (exec != null) exec.shutdownNow();
}
}
} }
} }