mirror of https://github.com/apache/lucene.git
LUCENE-10134: clean up the test from leaking threads and resources if an error occurs somewhere - this obscures the original cause of the problem.
This commit is contained in:
Dawid Weiss
parent
88b264a368
commit
d2b88b7a0b
|
|
@ -45,7 +45,6 @@ import org.apache.lucene.search.TermQuery;
|
|||
import org.apache.lucene.search.TopDocs;
|
||||
import org.apache.lucene.store.Directory;
|
||||
import org.apache.lucene.util.Accountable;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.NamedThreadFactory;
|
||||
import org.apache.lucene.util.TestUtil;
|
||||
|
|
@ -56,165 +55,162 @@ public class TestSortedSetDocValuesFacets extends FacetTestCase {
|
|||
// randomly uses SortedSetDV
|
||||
|
||||
public void testBasic() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
config.setMultiValued("a", true);
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "zoo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "baz"));
|
||||
writer.addDocument(config.build(doc));
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
}
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "zoo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "baz"));
|
||||
writer.addDocument(config.build(doc));
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
try {
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
assertEquals(
|
||||
"dim=a path=[] value=4 childCount=3\n foo (2)\n bar (1)\n zoo (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
assertEquals(
|
||||
"dim=b path=[] value=1 childCount=1\n baz (1)\n",
|
||||
facets.getTopChildren(10, "b").toString());
|
||||
|
||||
// DrillDown:
|
||||
DrillDownQuery q = new DrillDownQuery(config);
|
||||
q.add("a", "foo");
|
||||
q.add("b", "baz");
|
||||
TopDocs hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
} finally {
|
||||
if (exec != null) exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
assertEquals(
|
||||
"dim=a path=[] value=4 childCount=3\n foo (2)\n bar (1)\n zoo (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
assertEquals(
|
||||
"dim=b path=[] value=1 childCount=1\n baz (1)\n",
|
||||
facets.getTopChildren(10, "b").toString());
|
||||
|
||||
// DrillDown:
|
||||
DrillDownQuery q = new DrillDownQuery(config);
|
||||
q.add("a", "foo");
|
||||
q.add("b", "baz");
|
||||
TopDocs hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
if (exec != null) {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
writer.close();
|
||||
IOUtils.close(searcher.getIndexReader(), dir);
|
||||
}
|
||||
|
||||
// See: LUCENE-10070
|
||||
public void testCountAll() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("id", "0", Field.Store.NO));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new StringField("id", "0", Field.Store.NO));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
doc = new Document();
|
||||
doc.add(new StringField("id", "1", Field.Store.NO));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new StringField("id", "1", Field.Store.NO));
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.deleteDocuments(new Term("id", "0"));
|
||||
|
||||
writer.deleteDocuments(new Term("id", "0"));
|
||||
// NRT open
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
|
||||
// NRT open
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
Facets facets = new SortedSetDocValuesFacetCounts(state);
|
||||
|
||||
Facets facets = new SortedSetDocValuesFacetCounts(state);
|
||||
assertEquals(
|
||||
"dim=a path=[] value=1 childCount=1\n bar (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
|
||||
assertEquals(
|
||||
"dim=a path=[] value=1 childCount=1\n bar (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
ExecutorService exec =
|
||||
new ThreadPoolExecutor(
|
||||
1,
|
||||
TestUtil.nextInt(random(), 2, 6),
|
||||
Long.MAX_VALUE,
|
||||
TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(),
|
||||
new NamedThreadFactory("TestIndexSearcher"));
|
||||
try {
|
||||
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, exec);
|
||||
|
||||
ExecutorService exec =
|
||||
new ThreadPoolExecutor(
|
||||
1,
|
||||
TestUtil.nextInt(random(), 2, 6),
|
||||
Long.MAX_VALUE,
|
||||
TimeUnit.MILLISECONDS,
|
||||
new LinkedBlockingQueue<Runnable>(),
|
||||
new NamedThreadFactory("TestIndexSearcher"));
|
||||
|
||||
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, exec);
|
||||
|
||||
assertEquals(
|
||||
"dim=a path=[] value=1 childCount=1\n bar (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
|
||||
writer.close();
|
||||
IOUtils.close(searcher.getIndexReader(), dir);
|
||||
exec.shutdownNow();
|
||||
assertEquals(
|
||||
"dim=a path=[] value=1 childCount=1\n bar (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
} finally {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void testBasicSingleValued() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
config.setMultiValued("a", false);
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
}
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "baz"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
try {
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
assertEquals(
|
||||
"dim=a path=[] value=3 childCount=2\n foo (2)\n baz (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
assertEquals(
|
||||
"dim=b path=[] value=1 childCount=1\n bar (1)\n",
|
||||
facets.getTopChildren(10, "b").toString());
|
||||
|
||||
// DrillDown:
|
||||
DrillDownQuery q = new DrillDownQuery(config);
|
||||
q.add("a", "foo");
|
||||
q.add("b", "bar");
|
||||
TopDocs hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
} finally {
|
||||
if (exec != null) exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "baz"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
assertEquals(
|
||||
"dim=a path=[] value=3 childCount=2\n foo (2)\n baz (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
assertEquals(
|
||||
"dim=b path=[] value=1 childCount=1\n bar (1)\n",
|
||||
facets.getTopChildren(10, "b").toString());
|
||||
|
||||
// DrillDown:
|
||||
DrillDownQuery q = new DrillDownQuery(config);
|
||||
q.add("a", "foo");
|
||||
q.add("b", "bar");
|
||||
TopDocs hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
if (exec != null) {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
writer.close();
|
||||
IOUtils.close(searcher.getIndexReader(), dir);
|
||||
}
|
||||
|
||||
public void testDrillDownOptions() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
config.setDrillDownTermsIndexing("c", FacetsConfig.DrillDownTermsIndexing.NONE);
|
||||
config.setDrillDownTermsIndexing(
|
||||
|
|
@ -222,381 +218,377 @@ public class TestSortedSetDocValuesFacets extends FacetTestCase {
|
|||
config.setDrillDownTermsIndexing("e", FacetsConfig.DrillDownTermsIndexing.ALL_PATHS_NO_DIM);
|
||||
config.setDrillDownTermsIndexing("f", FacetsConfig.DrillDownTermsIndexing.FULL_PATH_ONLY);
|
||||
config.setDrillDownTermsIndexing("g", FacetsConfig.DrillDownTermsIndexing.ALL);
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("c", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("d", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("e", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("f", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("g", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("c", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("d", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("e", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("f", "foo"));
|
||||
doc.add(new SortedSetDocValuesFacetField("g", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
}
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
// Drill down with different indexing configuration options
|
||||
DrillDownQuery q = new DrillDownQuery(config);
|
||||
q.add("c");
|
||||
TopDocs hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("c", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("d");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("d", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("e");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("e", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("f");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("f", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("g");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("g", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
}
|
||||
}
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
|
||||
// Drill down with different indexing configuration options
|
||||
DrillDownQuery q = new DrillDownQuery(config);
|
||||
q.add("c");
|
||||
TopDocs hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("c", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("d");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("d", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("e");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("e", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("f");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(0, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("f", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("g");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
q = new DrillDownQuery(config);
|
||||
q.add("g", "foo");
|
||||
hits = searcher.search(q, 1);
|
||||
assertEquals(1, hits.totalHits.value);
|
||||
|
||||
if (exec != null) {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
writer.close();
|
||||
IOUtils.close(searcher.getIndexReader(), dir);
|
||||
}
|
||||
|
||||
// LUCENE-5090
|
||||
@SuppressWarnings("unused")
|
||||
public void testStaleState() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo"));
|
||||
writer.addDocument(config.build(doc));
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(r);
|
||||
|
||||
IndexReader r = writer.getReader();
|
||||
SortedSetDocValuesReaderState state = new DefaultSortedSetDocValuesReaderState(r);
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "baz"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "baz"));
|
||||
writer.addDocument(config.build(doc));
|
||||
try (IndexReader r2 = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r2);
|
||||
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
FacetsCollector c = new FacetsCollector();
|
||||
|
||||
FacetsCollector c = new FacetsCollector();
|
||||
searcher.search(new MatchAllDocsQuery(), c);
|
||||
|
||||
searcher.search(new MatchAllDocsQuery(), c);
|
||||
|
||||
expectThrows(
|
||||
IllegalStateException.class,
|
||||
() -> {
|
||||
new SortedSetDocValuesFacetCounts(state, c);
|
||||
});
|
||||
|
||||
r.close();
|
||||
writer.close();
|
||||
searcher.getIndexReader().close();
|
||||
dir.close();
|
||||
expectThrows(
|
||||
IllegalStateException.class,
|
||||
() -> {
|
||||
new SortedSetDocValuesFacetCounts(state, c);
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// LUCENE-5333
|
||||
public void testSparseFacets() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
}
|
||||
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo2"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "bar1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
}
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo3"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "bar2"));
|
||||
doc.add(new SortedSetDocValuesFacetField("c", "baz1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
try {
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
// Ask for top 10 labels for any dims that have counts:
|
||||
List<FacetResult> results = facets.getAllDims(10);
|
||||
|
||||
assertEquals(3, results.size());
|
||||
assertEquals(
|
||||
"dim=a path=[] value=3 childCount=3\n foo1 (1)\n foo2 (1)\n foo3 (1)\n",
|
||||
results.get(0).toString());
|
||||
assertEquals(
|
||||
"dim=b path=[] value=2 childCount=2\n bar1 (1)\n bar2 (1)\n",
|
||||
results.get(1).toString());
|
||||
assertEquals(
|
||||
"dim=c path=[] value=1 childCount=1\n baz1 (1)\n", results.get(2).toString());
|
||||
|
||||
Collection<Accountable> resources = state.getChildResources();
|
||||
assertTrue(state.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
|
||||
if (searcher.getIndexReader().leaves().size() > 1) {
|
||||
assertTrue(state.ramBytesUsed() > 0);
|
||||
assertFalse(resources.isEmpty());
|
||||
assertTrue(resources.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
|
||||
} else {
|
||||
assertEquals(0, state.ramBytesUsed());
|
||||
assertTrue(resources.isEmpty());
|
||||
}
|
||||
} finally {
|
||||
if (exec != null) exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo2"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "bar1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
if (random().nextBoolean()) {
|
||||
writer.commit();
|
||||
}
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo3"));
|
||||
doc.add(new SortedSetDocValuesFacetField("b", "bar2"));
|
||||
doc.add(new SortedSetDocValuesFacetField("c", "baz1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
|
||||
// NRT open
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
writer.close();
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
// Ask for top 10 labels for any dims that have counts:
|
||||
List<FacetResult> results = facets.getAllDims(10);
|
||||
|
||||
assertEquals(3, results.size());
|
||||
assertEquals(
|
||||
"dim=a path=[] value=3 childCount=3\n foo1 (1)\n foo2 (1)\n foo3 (1)\n",
|
||||
results.get(0).toString());
|
||||
assertEquals(
|
||||
"dim=b path=[] value=2 childCount=2\n bar1 (1)\n bar2 (1)\n", results.get(1).toString());
|
||||
assertEquals("dim=c path=[] value=1 childCount=1\n baz1 (1)\n", results.get(2).toString());
|
||||
|
||||
Collection<Accountable> resources = state.getChildResources();
|
||||
assertTrue(state.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
|
||||
if (searcher.getIndexReader().leaves().size() > 1) {
|
||||
assertTrue(state.ramBytesUsed() > 0);
|
||||
assertFalse(resources.isEmpty());
|
||||
assertTrue(resources.toString().contains(FacetsConfig.DEFAULT_INDEX_FIELD_NAME));
|
||||
} else {
|
||||
assertEquals(0, state.ramBytesUsed());
|
||||
assertTrue(resources.isEmpty());
|
||||
}
|
||||
|
||||
if (exec != null) {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
searcher.getIndexReader().close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testSomeSegmentsMissing() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo1"));
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
doc = new Document();
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
|
||||
doc = new Document();
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo2"));
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
|
||||
doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("a", "foo2"));
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
// NRT open
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
|
||||
// NRT open
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
writer.close();
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
try {
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
|
||||
// Ask for top 10 labels for any dims that have counts:
|
||||
assertEquals(
|
||||
"dim=a path=[] value=2 childCount=2\n foo1 (1)\n foo2 (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
|
||||
if (exec != null) {
|
||||
exec.shutdownNow();
|
||||
// Ask for top 10 labels for any dims that have counts:
|
||||
assertEquals(
|
||||
"dim=a path=[] value=2 childCount=2\n foo1 (1)\n foo2 (1)\n",
|
||||
facets.getTopChildren(10, "a").toString());
|
||||
} finally {
|
||||
if (exec != null) exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
searcher.getIndexReader().close();
|
||||
dir.close();
|
||||
}
|
||||
|
||||
public void testRandom() throws Exception {
|
||||
int fullIterations = LuceneTestCase.TEST_NIGHTLY ? 20 : 3;
|
||||
for (int fullIter = 0; fullIter < fullIterations; fullIter++) {
|
||||
String[] tokens = getRandomTokens(10);
|
||||
Directory indexDir = newDirectory();
|
||||
Directory taxoDir = newDirectory();
|
||||
|
||||
RandomIndexWriter w = new RandomIndexWriter(random(), indexDir);
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
int numDocs = atLeast(1000);
|
||||
// Most of the time allow up to 7 dims per doc, but occasionally limit all docs to a single
|
||||
// dim:
|
||||
int numDims;
|
||||
if (random().nextInt(10) < 8) {
|
||||
numDims = TestUtil.nextInt(random(), 1, 7);
|
||||
} else {
|
||||
numDims = 1;
|
||||
}
|
||||
List<TestDoc> testDocs = getRandomDocs(tokens, numDocs, numDims);
|
||||
for (TestDoc testDoc : testDocs) {
|
||||
Document doc = new Document();
|
||||
doc.add(newStringField("content", testDoc.content, Field.Store.NO));
|
||||
for (int j = 0; j < numDims; j++) {
|
||||
if (testDoc.dims[j] != null) {
|
||||
doc.add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j]));
|
||||
}
|
||||
}
|
||||
w.addDocument(config.build(doc));
|
||||
}
|
||||
|
||||
// NRT open
|
||||
IndexSearcher searcher = newSearcher(w.getReader());
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
|
||||
int iters = atLeast(100);
|
||||
for (int iter = 0; iter < iters; iter++) {
|
||||
String searchToken = tokens[random().nextInt(tokens.length)];
|
||||
if (VERBOSE) {
|
||||
System.out.println("\nTEST: iter content=" + searchToken);
|
||||
}
|
||||
FacetsCollector fc = new FacetsCollector();
|
||||
FacetsCollector.search(searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
|
||||
Facets facets;
|
||||
if (exec != null) {
|
||||
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, fc, exec);
|
||||
try (Directory indexDir = newDirectory();
|
||||
RandomIndexWriter w = new RandomIndexWriter(random(), indexDir)) {
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
int numDocs = atLeast(1000);
|
||||
// Most of the time allow up to 7 dims per doc, but occasionally limit all docs to a single
|
||||
// dim:
|
||||
int numDims;
|
||||
if (random().nextInt(10) < 8) {
|
||||
numDims = TestUtil.nextInt(random(), 1, 7);
|
||||
} else {
|
||||
facets = new SortedSetDocValuesFacetCounts(state, fc);
|
||||
numDims = 1;
|
||||
}
|
||||
|
||||
// Slow, yet hopefully bug-free, faceting:
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
Map<String, Integer>[] expectedCounts = new HashMap[numDims];
|
||||
for (int i = 0; i < numDims; i++) {
|
||||
expectedCounts[i] = new HashMap<>();
|
||||
}
|
||||
|
||||
for (TestDoc doc : testDocs) {
|
||||
if (doc.content.equals(searchToken)) {
|
||||
for (int j = 0; j < numDims; j++) {
|
||||
if (doc.dims[j] != null) {
|
||||
Integer v = expectedCounts[j].get(doc.dims[j]);
|
||||
if (v == null) {
|
||||
expectedCounts[j].put(doc.dims[j], 1);
|
||||
} else {
|
||||
expectedCounts[j].put(doc.dims[j], v.intValue() + 1);
|
||||
}
|
||||
}
|
||||
List<TestDoc> testDocs = getRandomDocs(tokens, numDocs, numDims);
|
||||
for (TestDoc testDoc : testDocs) {
|
||||
Document doc = new Document();
|
||||
doc.add(newStringField("content", testDoc.content, Field.Store.NO));
|
||||
for (int j = 0; j < numDims; j++) {
|
||||
if (testDoc.dims[j] != null) {
|
||||
doc.add(new SortedSetDocValuesFacetField("dim" + j, testDoc.dims[j]));
|
||||
}
|
||||
}
|
||||
w.addDocument(config.build(doc));
|
||||
}
|
||||
|
||||
List<FacetResult> expected = new ArrayList<>();
|
||||
for (int i = 0; i < numDims; i++) {
|
||||
List<LabelAndValue> labelValues = new ArrayList<>();
|
||||
int totCount = 0;
|
||||
for (Map.Entry<String, Integer> ent : expectedCounts[i].entrySet()) {
|
||||
labelValues.add(new LabelAndValue(ent.getKey(), ent.getValue()));
|
||||
totCount += ent.getValue();
|
||||
}
|
||||
sortLabelValues(labelValues);
|
||||
if (totCount > 0) {
|
||||
expected.add(
|
||||
new FacetResult(
|
||||
"dim" + i,
|
||||
new String[0],
|
||||
totCount,
|
||||
labelValues.toArray(new LabelAndValue[labelValues.size()]),
|
||||
labelValues.size()));
|
||||
// NRT open
|
||||
try (IndexReader r = w.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
|
||||
// Per-top-reader state:
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
try {
|
||||
int iters = atLeast(100);
|
||||
for (int iter = 0; iter < iters; iter++) {
|
||||
String searchToken = tokens[random().nextInt(tokens.length)];
|
||||
if (VERBOSE) {
|
||||
System.out.println("\nTEST: iter content=" + searchToken);
|
||||
}
|
||||
FacetsCollector fc = new FacetsCollector();
|
||||
FacetsCollector.search(
|
||||
searcher, new TermQuery(new Term("content", searchToken)), 10, fc);
|
||||
Facets facets;
|
||||
if (exec != null) {
|
||||
facets = new ConcurrentSortedSetDocValuesFacetCounts(state, fc, exec);
|
||||
} else {
|
||||
facets = new SortedSetDocValuesFacetCounts(state, fc);
|
||||
}
|
||||
|
||||
// Slow, yet hopefully bug-free, faceting:
|
||||
@SuppressWarnings({"rawtypes", "unchecked"})
|
||||
Map<String, Integer>[] expectedCounts = new HashMap[numDims];
|
||||
for (int i = 0; i < numDims; i++) {
|
||||
expectedCounts[i] = new HashMap<>();
|
||||
}
|
||||
|
||||
for (TestDoc doc : testDocs) {
|
||||
if (doc.content.equals(searchToken)) {
|
||||
for (int j = 0; j < numDims; j++) {
|
||||
if (doc.dims[j] != null) {
|
||||
Integer v = expectedCounts[j].get(doc.dims[j]);
|
||||
if (v == null) {
|
||||
expectedCounts[j].put(doc.dims[j], 1);
|
||||
} else {
|
||||
expectedCounts[j].put(doc.dims[j], v.intValue() + 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
List<FacetResult> expected = new ArrayList<>();
|
||||
for (int i = 0; i < numDims; i++) {
|
||||
List<LabelAndValue> labelValues = new ArrayList<>();
|
||||
int totCount = 0;
|
||||
for (Map.Entry<String, Integer> ent : expectedCounts[i].entrySet()) {
|
||||
labelValues.add(new LabelAndValue(ent.getKey(), ent.getValue()));
|
||||
totCount += ent.getValue();
|
||||
}
|
||||
sortLabelValues(labelValues);
|
||||
if (totCount > 0) {
|
||||
expected.add(
|
||||
new FacetResult(
|
||||
"dim" + i,
|
||||
new String[0],
|
||||
totCount,
|
||||
labelValues.toArray(new LabelAndValue[labelValues.size()]),
|
||||
labelValues.size()));
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by highest value, tie break by value:
|
||||
sortFacetResults(expected);
|
||||
|
||||
List<FacetResult> actual = facets.getAllDims(10);
|
||||
|
||||
// Messy: fixup ties
|
||||
// sortTies(actual);
|
||||
|
||||
assertEquals(expected, actual);
|
||||
}
|
||||
} finally {
|
||||
if (exec != null) exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by highest value, tie break by value:
|
||||
sortFacetResults(expected);
|
||||
|
||||
List<FacetResult> actual = facets.getAllDims(10);
|
||||
|
||||
// Messy: fixup ties
|
||||
// sortTies(actual);
|
||||
|
||||
assertEquals(expected, actual);
|
||||
}
|
||||
|
||||
if (exec != null) {
|
||||
exec.shutdownNow();
|
||||
}
|
||||
w.close();
|
||||
IOUtils.close(searcher.getIndexReader(), indexDir, taxoDir);
|
||||
}
|
||||
}
|
||||
|
||||
public void testNonExistentDimension() throws Exception {
|
||||
Directory dir = newDirectory();
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
|
||||
try (Directory dir = newDirectory();
|
||||
RandomIndexWriter writer = new RandomIndexWriter(random(), dir)) {
|
||||
FacetsConfig config = new FacetsConfig();
|
||||
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("foo", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
Document doc = new Document();
|
||||
doc.add(new SortedSetDocValuesFacetField("foo", "bar"));
|
||||
writer.addDocument(config.build(doc));
|
||||
writer.commit();
|
||||
|
||||
IndexSearcher searcher = newSearcher(writer.getReader());
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
try (IndexReader r = writer.getReader()) {
|
||||
IndexSearcher searcher = newSearcher(r);
|
||||
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
FacetResult result = facets.getTopChildren(5, "non-existent dimension");
|
||||
SortedSetDocValuesReaderState state =
|
||||
new DefaultSortedSetDocValuesReaderState(searcher.getIndexReader());
|
||||
|
||||
// make sure the result is null (and no exception was thrown)
|
||||
assertNull(result);
|
||||
ExecutorService exec = randomExecutorServiceOrNull();
|
||||
try {
|
||||
Facets facets = getAllFacets(searcher, state, exec);
|
||||
FacetResult result = facets.getTopChildren(5, "non-existent dimension");
|
||||
|
||||
writer.close();
|
||||
IOUtils.close(searcher.getIndexReader(), dir);
|
||||
if (exec != null) {
|
||||
exec.shutdownNow();
|
||||
// make sure the result is null (and no exception was thrown)
|
||||
assertNull(result);
|
||||
} finally {
|
||||
if (exec != null) exec.shutdownNow();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue