iter = categories.iterator();
+ for (int i = 0; i < upto; i++) {
+ int ordinal = ordinals.ints[i];
+ CategoryPath cp = iter.next();
+ OrdinalPolicy op = clp.getOrdinalPolicy(cp.components[0]);
+ if (op != OrdinalPolicy.NO_PARENTS) {
+ // need to add parents too
int parent = taxoWriter.getParent(ordinal);
while (parent > 0) {
ordinals.ints[ordinals.length++] = parent;
parent = taxoWriter.getParent(parent);
}
+ if (op == OrdinalPolicy.ALL_BUT_DIMENSION) { // discard the last added parent, which is the dimension
+ ordinals.length--;
+ }
}
}
return ordinalsEncoder.encode(ordinals);
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
index 08c1747ed7b..dcf8ed191a2 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
@@ -4,6 +4,7 @@ import java.io.IOException;
import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.DocValuesCategoryListIterator;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.util.PartitionsUtils;
import org.apache.lucene.util.encoding.DGapVInt8IntEncoder;
import org.apache.lucene.util.encoding.IntDecoder;
@@ -35,25 +36,61 @@ import org.apache.lucene.util.encoding.UniqueValuesIntEncoder;
*/
public class CategoryListParams {
- /** OrdinalPolicy defines which ordinals are encoded for every document. */
+ /**
+ * Defines which category ordinals are encoded for every document. This also
+ * affects how category ordinals are aggregated, check the different policies
+ * for more details.
+ */
public static enum OrdinalPolicy {
/**
- * Encodes only the ordinal of leaf nodes. That is, the category A/B/C will
- * not encode the ordinals of A and A/B.
+ * Encodes only the ordinals of leaf nodes. That is, for the category A/B/C,
+ * the ordinals of A and A/B will not be encoded. This policy is efficient
+ * for hierarchical dimensions, as it reduces the number of ordinals that
+ * are visited per document. During faceted search, this policy behaves
+ * exactly like {@link #ALL_PARENTS}, and the counts of all path components
+ * will be computed as well.
*
*
* NOTE: this {@link OrdinalPolicy} requires a special collector or
- * accumulator, which will fix the parents' counts, unless you are not
- * interested in the parents counts.
+ * accumulator, which will fix the parents' counts.
+ *
+ *
+ * NOTE: since only leaf nodes are encoded for the document, you
+ * should use this policy when the same document doesn't share two
+ * categories that have a mutual parent, or otherwise the counts will be
+ * wrong (the mutual parent will be over-counted). For example, if a
+ * document has the categories A/B/C and A/B/D, then with this policy the
+ * counts of "A" and "B" will be 2, which is wrong. If you intend to index
+ * hierarchical dimensions, with more than one category per document, you
+ * should use either {@link #ALL_PARENTS} or {@link #ALL_BUT_DIMENSION}.
*/
NO_PARENTS,
/**
* Encodes the ordinals of all path components. That is, the category A/B/C
- * will encode the ordinals of A and A/B as well. This is the default
- * {@link OrdinalPolicy}.
+ * will encode the ordinals of A and A/B as well. If you don't require the
+ * dimension's count during search, consider using
+ * {@link #ALL_BUT_DIMENSION}.
*/
- ALL_PARENTS
+ ALL_PARENTS,
+
+ /**
+ * Encodes the ordinals of all path components except the dimension. The
+ * dimension of a category is defined to be the first components in
+ * {@link CategoryPath#components}. For the category A/B/C, the ordinal of
+ * A/B will be encoded as well, however not the ordinal of A.
+ *
+ *
+ * NOTE: when facets are aggregated, this policy behaves exactly like
+ * {@link #ALL_PARENTS}, except that the dimension is never counted. I.e. if
+ * you ask to count the facet "A", then while in {@link #ALL_PARENTS} you
+ * will get counts for "A" and its children, with this policy you
+ * will get counts for only its children. This policy is the default
+ * one, and makes sense for using with flat dimensions, whenever your
+ * application does not require the dimension's count. Otherwise, use
+ * {@link #ALL_PARENTS}.
+ */
+ ALL_BUT_DIMENSION
}
/** The default field used to store the facets information. */
@@ -63,7 +100,7 @@ public class CategoryListParams {
* The default {@link OrdinalPolicy} that's used when encoding a document's
* category ordinals.
*/
- public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_PARENTS;
+ public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_BUT_DIMENSION;
public final String field;
@@ -115,19 +152,15 @@ public class CategoryListParams {
return false;
}
CategoryListParams other = (CategoryListParams) o;
- if (this.hashCode != other.hashCode) {
+ if (hashCode != other.hashCode) {
return false;
}
-
- // The above hashcodes might equal each other in the case of a collision,
- // so at this point only directly term equality testing will settle
- // the equality test.
return field.equals(other.field);
}
@Override
public int hashCode() {
- return this.hashCode;
+ return hashCode;
}
/** Create the {@link CategoryListIterator} for the specified partition. */
@@ -137,14 +170,18 @@ public class CategoryListParams {
return new DocValuesCategoryListIterator(docValuesField, createEncoder().createMatchingDecoder());
}
- /** Returns the {@link OrdinalPolicy} to use for this {@link CategoryListParams}. */
- public OrdinalPolicy getOrdinalPolicy() {
+ /**
+ * Returns the {@link OrdinalPolicy} to use for the given dimension. This
+ * {@link CategoryListParams} always returns {@link #DEFAULT_ORDINAL_POLICY}
+ * for all dimensions.
+ */
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
return DEFAULT_ORDINAL_POLICY;
}
@Override
public String toString() {
- return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy();
+ return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy(null);
}
-}
\ No newline at end of file
+}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionOrdinalPolicy.java b/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionOrdinalPolicy.java
new file mode 100644
index 00000000000..c91b1a69ccf
--- /dev/null
+++ b/lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionOrdinalPolicy.java
@@ -0,0 +1,55 @@
+package org.apache.lucene.facet.index.params;
+
+import java.util.Map;
+
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A {@link CategoryListParams} which allow controlling the
+ * {@link CategoryListParams.OrdinalPolicy} used for each dimension. The
+ * dimension is specified as the first component in
+ * {@link CategoryPath#components}.
+ */
+public class PerDimensionOrdinalPolicy extends CategoryListParams {
+
+ private final Map policies;
+ private final OrdinalPolicy defaultOP;
+
+ public PerDimensionOrdinalPolicy(Map policies) {
+ this(policies, DEFAULT_ORDINAL_POLICY);
+ }
+
+ public PerDimensionOrdinalPolicy(Map policies, OrdinalPolicy defaultOP) {
+ this.defaultOP = defaultOP;
+ this.policies = policies;
+ }
+
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ OrdinalPolicy op = policies.get(dimension);
+ return op == null ? defaultOP : op;
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " policies=" + policies;
+ }
+
+}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
index 613bfcb7b4e..23a97e3cd76 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
@@ -81,7 +81,7 @@ import org.apache.lucene.util.encoding.DGapVInt8IntDecoder;
public class CountingFacetsCollector extends FacetsCollector {
private final FacetSearchParams fsp;
- private final OrdinalPolicy ordinalPolicy;
+ private final CategoryListParams clp;
private final TaxonomyReader taxoReader;
private final BytesRef buf = new BytesRef(32);
private final FacetArrays facetArrays;
@@ -101,8 +101,7 @@ public class CountingFacetsCollector extends FacetsCollector {
assert assertParams(fsp) == null : assertParams(fsp);
this.fsp = fsp;
- CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
- this.ordinalPolicy = clp.getOrdinalPolicy();
+ this.clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
this.facetsField = clp.field;
this.taxoReader = taxoReader;
this.facetArrays = facetArrays;
@@ -209,21 +208,21 @@ public class CountingFacetsCollector extends FacetsCollector {
}
}
- private void countParents(int[] parents) {
- // counts[0] is the count of ROOT, which we don't care about and counts[1]
- // can only update counts[0], so we don't bother to visit it too. also,
- // since parents always have lower ordinals than their children, we traverse
- // the array backwards. this also allows us to update just the immediate
- // parent's count (actually, otherwise it would be a mistake).
- for (int i = counts.length - 1; i > 1; i--) {
- int count = counts[i];
- if (count > 0) {
- int parent = parents[i];
- if (parent != 0) {
- counts[parent] += count;
- }
- }
+ /**
+ * Computes the counts of ordinals under the given ordinal's tree, by
+ * recursively going down to leaf nodes and rollin up their counts (called
+ * only with categories are indexing with OrdinalPolicy.NO_PARENTS).
+ */
+ private int rollupCounts(int ordinal, int[] children, int[] siblings) {
+ int count = 0;
+ while (ordinal != TaxonomyReader.INVALID_ORDINAL) {
+ int childCount = counts[ordinal];
+ childCount += rollupCounts(children[ordinal], children, siblings);
+ counts[ordinal] = childCount;
+ count += childCount;
+ ordinal = siblings[ordinal];
}
+ return count;
}
@Override
@@ -234,11 +233,6 @@ public class CountingFacetsCollector extends FacetsCollector {
ParallelTaxonomyArrays arrays = taxoReader.getParallelTaxonomyArrays();
- if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
- // need to count parents
- countParents(arrays.parents());
- }
-
// compute top-K
final int[] children = arrays.children();
final int[] siblings = arrays.siblings();
@@ -248,6 +242,12 @@ public class CountingFacetsCollector extends FacetsCollector {
if (rootOrd == TaxonomyReader.INVALID_ORDINAL) { // category does not exist
continue;
}
+ OrdinalPolicy ordinalPolicy = clp.getOrdinalPolicy(fr.categoryPath.components[0]);
+ if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
+ // need to count parents
+ counts[rootOrd] += rollupCounts(children[rootOrd], children, siblings);
+ }
+
FacetResultNode root = new FacetResultNode();
root.ordinal = rootOrd;
root.label = fr.categoryPath;
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
index ad8a44d454e..1809d0c4cf5 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
@@ -43,13 +43,21 @@ public abstract class FacetsCollector extends Collector {
* Returns the most optimized {@link FacetsCollector} for the given search
* parameters. The returned {@link FacetsCollector} is guaranteed to satisfy
* the requested parameters.
+ *
+ * @throws IllegalArgumentException
+ * if there is no built-in collector that can satisfy the search
+ * parameters.
*/
public static FacetsCollector create(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
if (CountingFacetsCollector.assertParams(fsp) == null) {
return new CountingFacetsCollector(fsp, taxoReader);
}
- return new StandardFacetsCollector(fsp, indexReader, taxoReader);
+ if (StandardFacetsCollector.assertParams(fsp) == null) {
+ return new StandardFacetsCollector(fsp, indexReader, taxoReader);
+ }
+
+ throw new IllegalArgumentException("None of the built-in FacetsCollectors can handle the given search params");
}
/**
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java b/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java
index b93320d055b..5147b35bf2f 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java
@@ -49,6 +49,17 @@ public class StandardFacetsCollector extends FacetsCollector {
private List results;
private Object resultsGuard;
+ static String assertParams(FacetSearchParams fsp) {
+ // make sure none of the categories in the given FacetRequests was indexed with NO_PARENTS
+ for (FacetRequest fr : fsp.facetRequests) {
+ CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fr.categoryPath);
+ if (clp.getOrdinalPolicy(fr.categoryPath.components[0]) == OrdinalPolicy.NO_PARENTS) {
+ return "this collector does not support aggregating categories that were indexed with OrdinalPolicy.NO_PARENTS";
+ }
+ }
+ return null;
+ }
+
/**
* Create a collector for accumulating facets while collecting documents
* during search.
@@ -62,6 +73,7 @@ public class StandardFacetsCollector extends FacetsCollector {
* taxonomy containing the facets.
*/
public StandardFacetsCollector(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
+ assert assertParams(facetSearchParams) == null : assertParams(facetSearchParams);
facetsAccumulator = initFacetsAccumulator(facetSearchParams, indexReader, taxonomyReader);
scoreDocIdCollector = initScoredDocCollector(facetSearchParams, indexReader, taxonomyReader);
resultsGuard = new Object();
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java b/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java
index 923045ff2d9..917c4ab1ddb 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java
@@ -74,6 +74,7 @@ public class ScoredDocIdsUtils {
/** Clear all deleted documents from a given open-bit-set according to a given reader */
private static void clearDeleted(final IndexReader reader, final FixedBitSet set) throws IOException {
+ // TODO use BitsFilteredDocIdSet?
// If there are no deleted docs
if (!reader.hasDeletions()) {
diff --git a/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java b/lucene/facet/src/java/org/apache/lucene/facet/util/TaxonomyMergeUtils.java
similarity index 94%
rename from lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
rename to lucene/facet/src/java/org/apache/lucene/facet/util/TaxonomyMergeUtils.java
index d48ea1109bd..0e157bbbe66 100644
--- a/lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/util/TaxonomyMergeUtils.java
@@ -1,9 +1,8 @@
-package org.apache.lucene.facet.example.merge;
+package org.apache.lucene.facet.util;
import java.io.IOException;
import java.util.List;
-import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.index.OrdinalMappingAtomicReader;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
@@ -17,6 +16,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Version;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -36,6 +36,7 @@ import org.apache.lucene.store.Directory;
*/
/**
+ * Utility methods for merging index and taxonomy directories.
* @lucene.experimental
*/
public class TaxonomyMergeUtils {
@@ -52,8 +53,7 @@ public class TaxonomyMergeUtils {
*/
public static void merge(Directory srcIndexDir, Directory srcTaxDir, Directory destIndexDir, Directory destTaxDir,
FacetIndexingParams params) throws IOException {
- IndexWriter destIndexWriter = new IndexWriter(destIndexDir,
- new IndexWriterConfig(ExampleUtils.EXAMPLE_VER, null));
+ IndexWriter destIndexWriter = new IndexWriter(destIndexDir, new IndexWriterConfig(Version.LUCENE_42, null));
DirectoryTaxonomyWriter destTaxWriter = new DirectoryTaxonomyWriter(destTaxDir);
merge(srcIndexDir, srcTaxDir, new MemoryOrdinalMap(), destIndexWriter, destTaxWriter, params);
destTaxWriter.close();
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
index 91df84c6a0f..2ad38f01e78 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
@@ -6,6 +6,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -17,6 +18,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -44,6 +46,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.collections.IntToObjectMap;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -73,7 +76,8 @@ public abstract class FacetTestBase extends FacetTestCase {
SearchTaxoDirPair() {}
}
- private static HashMap dirsPerPartitionSize;
+ private static IntToObjectMap dirsPerPartitionSize;
+ private static IntToObjectMap fipPerPartitionSize;
private static File TEST_DIR;
/** Documents text field. */
@@ -91,12 +95,15 @@ public abstract class FacetTestBase extends FacetTestCase {
@BeforeClass
public static void beforeClassFacetTestBase() {
TEST_DIR = _TestUtil.getTempDir("facets");
- dirsPerPartitionSize = new HashMap();
+ dirsPerPartitionSize = new IntToObjectMap();
+ fipPerPartitionSize = new IntToObjectMap();
}
@AfterClass
public static void afterClassFacetTestBase() throws Exception {
- for (SearchTaxoDirPair pair : dirsPerPartitionSize.values()) {
+ Iterator iter = dirsPerPartitionSize.iterator();
+ while (iter.hasNext()) {
+ SearchTaxoDirPair pair = iter.next();
IOUtils.close(pair.searchDir, pair.taxoDir);
}
}
@@ -128,20 +135,16 @@ public abstract class FacetTestBase extends FacetTestCase {
return DEFAULT_CONTENT[doc];
}
- /** Prepare index (in RAM) with single partition */
- protected final void initIndex() throws Exception {
- initIndex(Integer.MAX_VALUE);
- }
-
- /** Prepare index (in RAM) with some documents and some facets */
- protected final void initIndex(int partitionSize) throws Exception {
- initIndex(partitionSize, false);
+ /** Prepare index (in RAM) with some documents and some facets. */
+ protected final void initIndex(FacetIndexingParams fip) throws Exception {
+ initIndex(false, fip);
}
- /** Prepare index (in RAM/Disk) with some documents and some facets */
- protected final void initIndex(int partitionSize, boolean forceDisk) throws Exception {
+ /** Prepare index (in RAM/Disk) with some documents and some facets. */
+ protected final void initIndex(boolean forceDisk, FacetIndexingParams fip) throws Exception {
+ int partitionSize = fip.getPartitionSize();
if (VERBOSE) {
- System.out.println("Partition Size: " + partitionSize+" forceDisk: "+forceDisk);
+ System.out.println("Partition Size: " + partitionSize + " forceDisk: "+forceDisk);
}
SearchTaxoDirPair pair = dirsPerPartitionSize.get(Integer.valueOf(partitionSize));
@@ -158,7 +161,7 @@ public abstract class FacetTestBase extends FacetTestCase {
RandomIndexWriter iw = new RandomIndexWriter(random(), pair.searchDir, getIndexWriterConfig(getAnalyzer()));
TaxonomyWriter taxo = new DirectoryTaxonomyWriter(pair.taxoDir, OpenMode.CREATE);
- populateIndex(iw, taxo, getFacetIndexingParams(partitionSize));
+ populateIndex(iw, taxo, fip);
// commit changes (taxonomy prior to search index for consistency)
taxo.commit();
@@ -182,14 +185,40 @@ public abstract class FacetTestBase extends FacetTestCase {
/** Returns a {@link FacetIndexingParams} per the given partition size. */
protected FacetIndexingParams getFacetIndexingParams(final int partSize) {
- // several of our encoders don't support the value 0,
- // which is one of the values encoded when dealing w/ partitions.
- return new FacetIndexingParams() {
- @Override
- public int getPartitionSize() {
- return partSize;
- }
- };
+ return getFacetIndexingParams(partSize, false);
+ }
+
+ /**
+ * Returns a {@link FacetIndexingParams} per the given partition size. If
+ * requested, then {@link OrdinalPolicy} will be set to
+ * {@link OrdinalPolicy#ALL_PARENTS}, otherwise it will randomize.
+ */
+ protected FacetIndexingParams getFacetIndexingParams(final int partSize, final boolean forceAllParents) {
+ FacetIndexingParams fip = fipPerPartitionSize.get(partSize);
+ if (fip == null) {
+ // randomize OrdinalPolicy. Since not all Collectors / Accumulators
+ // support NO_PARENTS, don't include it.
+ // TODO: once all code paths support NO_PARENTS, randomize it too.
+ CategoryListParams randomOP = new CategoryListParams() {
+ final OrdinalPolicy op = random().nextBoolean() ? OrdinalPolicy.ALL_BUT_DIMENSION : OrdinalPolicy.ALL_PARENTS;
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return forceAllParents ? OrdinalPolicy.ALL_PARENTS : op;
+ }
+ };
+
+ // several of our encoders don't support the value 0,
+ // which is one of the values encoded when dealing w/ partitions,
+ // therefore don't randomize the encoder.
+ fip = new FacetIndexingParams(randomOP) {
+ @Override
+ public int getPartitionSize() {
+ return partSize;
+ }
+ };
+ fipPerPartitionSize.put(partSize, fip);
+ }
+ return fip;
}
/**
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
index 7846f154e25..aef675e7da5 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
@@ -1,19 +1,7 @@
package org.apache.lucene.facet;
-import java.io.IOException;
-
-import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -34,68 +22,6 @@ import org.apache.lucene.util.LuceneTestCase;
public class FacetTestUtils {
- public static class IndexTaxonomyReaderPair {
- public DirectoryReader indexReader;
- public DirectoryTaxonomyReader taxReader;
- public IndexSearcher indexSearcher;
-
- public void close() throws IOException {
- indexReader.close();
- taxReader.close();
- }
-
- }
-
- public static class IndexTaxonomyWriterPair {
- public IndexWriter indexWriter;
- public TaxonomyWriter taxWriter;
-
- public void close() throws IOException {
- indexWriter.close();
- taxWriter.close();
- }
-
- public void commit() throws IOException {
- indexWriter.commit();
- taxWriter.commit();
- }
- }
-
- public static Directory[][] createIndexTaxonomyDirs(int number) {
- Directory[][] dirs = new Directory[number][2];
- for (int i = 0; i < number; i++) {
- dirs[i][0] = LuceneTestCase.newDirectory();
- dirs[i][1] = LuceneTestCase.newDirectory();
- }
- return dirs;
- }
-
- public static IndexTaxonomyReaderPair[] createIndexTaxonomyReaderPair(Directory[][] dirs) throws IOException {
- IndexTaxonomyReaderPair[] pairs = new IndexTaxonomyReaderPair[dirs.length];
- for (int i = 0; i < dirs.length; i++) {
- IndexTaxonomyReaderPair pair = new IndexTaxonomyReaderPair();
- pair.indexReader = DirectoryReader.open(dirs[i][0]);
- pair.indexSearcher = new IndexSearcher(pair.indexReader);
- pair.taxReader = new DirectoryTaxonomyReader(dirs[i][1]);
- pairs[i] = pair;
- }
- return pairs;
- }
-
- public static IndexTaxonomyWriterPair[] createIndexTaxonomyWriterPair(Directory[][] dirs) throws IOException {
- IndexTaxonomyWriterPair[] pairs = new IndexTaxonomyWriterPair[dirs.length];
- for (int i = 0; i < dirs.length; i++) {
- IndexTaxonomyWriterPair pair = new IndexTaxonomyWriterPair();
- pair.indexWriter = new IndexWriter(dirs[i][0], new IndexWriterConfig(
- LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(LuceneTestCase.random())));
- pair.taxWriter = new DirectoryTaxonomyWriter(dirs[i][1]);
- pair.indexWriter.commit();
- pair.taxWriter.commit();
- pairs[i] = pair;
- }
- return pairs;
- }
-
public static String toSimpleString(FacetResult fr) {
StringBuilder sb = new StringBuilder();
toSimpleString(0, sb, fr.getFacetResultNode(), "");
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java b/lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java
index 8f311f0991f..698d41a06bf 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java
@@ -8,7 +8,6 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.example.merge.TaxonomyMergeUtils;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
@@ -18,6 +17,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.facet.util.TaxonomyMergeUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java b/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
index f45aba5c4f3..9c5c477bbf3 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
@@ -25,6 +25,8 @@ import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
+import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.DrillDown;
import org.apache.lucene.facet.search.FacetsCollector;
@@ -368,9 +370,23 @@ public class TestFacetsPayloadMigrationReader extends FacetTestCase {
// set custom CLP fields for two dimensions and use the default ($facets) for the other two
HashMap params = new HashMap();
- params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]));
- params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]));
- FacetIndexingParams fip = new PerDimensionIndexingParams(params) {
+ params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]) {
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return OrdinalPolicy.ALL_PARENTS;
+ }
+ });
+ params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]) {
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return OrdinalPolicy.ALL_PARENTS;
+ }
+ });
+
+ HashMap policies = new HashMap();
+ policies.put(DIMENSIONS[2], OrdinalPolicy.ALL_PARENTS);
+ policies.put(DIMENSIONS[3], OrdinalPolicy.ALL_PARENTS);
+ FacetIndexingParams fip = new PerDimensionIndexingParams(params, new PerDimensionOrdinalPolicy(policies)) {
@Override
public int getPartitionSize() {
return partitionSize;
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java b/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
index 45e2322bbd5..f9f58d9b879 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
@@ -51,10 +51,9 @@ public abstract class BaseTestTopK extends FacetTestBase {
private int nextInt;
@Override
- protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo,
- FacetIndexingParams iParams) throws IOException {
+ protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo, FacetIndexingParams fip) throws IOException {
currDoc = -1;
- super.populateIndex(iw, taxo, iParams);
+ super.populateIndex(iw, taxo, fip);
}
/** prepare the next random int */
@@ -94,17 +93,13 @@ public abstract class BaseTestTopK extends FacetTestBase {
return Arrays.asList(cp);
}
- protected FacetSearchParams searchParamsWithRequests(int numResults) {
- return searchParamsWithRequests(numResults, Integer.MAX_VALUE);
- }
-
- protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
+ protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
List facetRequests = new ArrayList();
facetRequests.add(new CountFacetRequest(new CategoryPath("a"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "2", "26", "267"), numResults));
- return getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
+ return getFacetSearchParams(facetRequests, fip);
}
@Override
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java
index 3904483751e..1f7a695c9c0 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java
@@ -16,8 +16,9 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest.SortBy;
@@ -146,13 +147,11 @@ public class CountingFacetsCollectorTest extends FacetTestCase {
termExpectedCounts.put(cp, termExpectedCounts.get(cp) + 1);
}
}
- // add 1 to each dimension
- allExpectedCounts.put(CP_A, allExpectedCounts.get(CP_A) + 1);
+ // add 1 to each NO_PARENTS dimension
allExpectedCounts.put(CP_B, allExpectedCounts.get(CP_B) + 1);
allExpectedCounts.put(CP_C, allExpectedCounts.get(CP_C) + 1);
allExpectedCounts.put(CP_D, allExpectedCounts.get(CP_D) + 1);
if (updateTermExpectedCounts) {
- termExpectedCounts.put(CP_A, termExpectedCounts.get(CP_A) + 1);
termExpectedCounts.put(CP_B, termExpectedCounts.get(CP_B) + 1);
termExpectedCounts.put(CP_C, termExpectedCounts.get(CP_C) + 1);
termExpectedCounts.put(CP_D, termExpectedCounts.get(CP_D) + 1);
@@ -252,19 +251,13 @@ public class CountingFacetsCollectorTest extends FacetTestCase {
conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges, so we can control the index segments
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
- CategoryListParams allParents = new CategoryListParams();
- CategoryListParams noParents = new CategoryListParams("no_parents") {
- @Override
- public OrdinalPolicy getOrdinalPolicy() {
- return OrdinalPolicy.NO_PARENTS;
- }
- };
- Map params = new HashMap();
- params.put(CP_A, allParents);
- params.put(CP_B, allParents);
- params.put(CP_C, noParents);
- params.put(CP_D, noParents);
- fip = new PerDimensionIndexingParams(params);
+
+ Map policies = new HashMap();
+ policies.put(CP_B.components[0], OrdinalPolicy.ALL_PARENTS);
+ policies.put(CP_C.components[0], OrdinalPolicy.NO_PARENTS);
+ policies.put(CP_D.components[0], OrdinalPolicy.NO_PARENTS);
+ CategoryListParams clp = new PerDimensionOrdinalPolicy(policies);
+ fip = new FacetIndexingParams(clp);
allExpectedCounts = newCounts();
termExpectedCounts = newCounts();
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
index a4f53e7da0a..1e402cc096c 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
@@ -104,9 +104,9 @@ public class TestDemoFacets extends FacetTestCase {
// Retrieve & verify results:
List results = c.getFacetResults();
assertEquals(2, results.size());
- assertEquals("Publish Date (5)\n 2012 (2)\n 2010 (2)\n 1999 (1)\n",
+ assertEquals("Publish Date (0)\n 2012 (2)\n 2010 (2)\n 1999 (1)\n",
FacetTestUtils.toSimpleString(results.get(0)));
- assertEquals("Author (5)\n Lisa (2)\n Frank (1)\n Susan (1)\n Bob (1)\n",
+ assertEquals("Author (0)\n Lisa (2)\n Frank (1)\n Susan (1)\n Bob (1)\n",
FacetTestUtils.toSimpleString(results.get(1)));
@@ -117,7 +117,7 @@ public class TestDemoFacets extends FacetTestCase {
searcher.search(q2, c);
results = c.getFacetResults();
assertEquals(1, results.size());
- assertEquals("Author (2)\n Lisa (1)\n Bob (1)\n",
+ assertEquals("Author (0)\n Lisa (1)\n Bob (1)\n",
FacetTestUtils.toSimpleString(results.get(0)));
// Smoke test PrintTaxonomyStats:
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
index 13fc3d18dcf..a93d2611148 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
@@ -14,6 +14,7 @@ import org.junit.Before;
import org.junit.Test;
import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.ScoredDocIDs;
import org.apache.lucene.facet.search.ScoredDocIdCollector;
@@ -48,11 +49,14 @@ import org.apache.lucene.facet.taxonomy.CategoryPath;
*/
public class TestFacetsAccumulatorWithComplement extends FacetTestBase {
+ private FacetIndexingParams fip;
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
- initIndex();
+ fip = getFacetIndexingParams(Integer.MAX_VALUE);
+ initIndex(fip);
}
@Override
@@ -125,7 +129,7 @@ public class TestFacetsAccumulatorWithComplement extends FacetTestBase {
/** compute facets with certain facet requests and docs */
private List findFacets(ScoredDocIDs sDocids, boolean withComplement) throws IOException {
- FacetSearchParams fsp = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(new CategoryPath("root","a"), 10));
+ FacetSearchParams fsp = new FacetSearchParams(fip, new CountFacetRequest(new CategoryPath("root","a"), 10));
FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(fsp, indexReader, taxoReader);
fAccumulator.setComplementThreshold(
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
index a0da35d6616..12252fa91d9 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
@@ -14,7 +14,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.FacetTestUtils;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
@@ -77,12 +76,14 @@ public class TestMultipleCategoryLists extends FacetTestCase {
@Test
public void testDefault() throws Exception {
- Directory[][] dirs = getDirs();
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
// create and open an index writer
- RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+ RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
- TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+ TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
PerDimensionIndexingParams iParams = new PerDimensionIndexingParams(Collections.emptyMap());
@@ -92,7 +93,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
- TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+ TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@@ -105,17 +106,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$facets", ir);
IOUtils.close(tr, ir, iw, tw);
- IOUtils.close(dirs[0]);
+ IOUtils.close(indexDir, taxoDir);
}
@Test
public void testCustom() throws Exception {
- Directory[][] dirs = getDirs();
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
// create and open an index writer
- RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+ RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
- TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+ TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
PerDimensionIndexingParams iParams = new PerDimensionIndexingParams(
Collections.singletonMap(new CategoryPath("Author"), new CategoryListParams("$author")));
@@ -125,7 +128,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
- TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+ TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@@ -139,17 +142,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$author", ir);
IOUtils.close(tr, ir, iw, tw);
- IOUtils.close(dirs[0]);
+ IOUtils.close(indexDir, taxoDir);
}
@Test
public void testTwoCustomsSameField() throws Exception {
- Directory[][] dirs = getDirs();
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
// create and open an index writer
- RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+ RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
- TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+ TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
Map paramsMap = new HashMap();
paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$music"));
@@ -161,7 +166,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
- TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+ TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@@ -176,7 +181,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$music", ir);
IOUtils.close(tr, ir, iw, tw);
- IOUtils.close(dirs[0]);
+ IOUtils.close(indexDir, taxoDir);
}
private void assertOrdinalsExist(String field, IndexReader ir) throws IOException {
@@ -191,12 +196,14 @@ public class TestMultipleCategoryLists extends FacetTestCase {
@Test
public void testDifferentFieldsAndText() throws Exception {
- Directory[][] dirs = getDirs();
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
// create and open an index writer
- RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+ RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
- TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+ TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
Map paramsMap = new HashMap();
paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$bands"));
@@ -208,7 +215,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
- TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+ TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@@ -222,17 +229,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$composers", ir);
IOUtils.close(tr, ir, iw, tw);
- IOUtils.close(dirs[0]);
+ IOUtils.close(indexDir, taxoDir);
}
@Test
public void testSomeSameSomeDifferent() throws Exception {
- Directory[][] dirs = getDirs();
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
// create and open an index writer
- RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+ RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
- TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+ TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
Map paramsMap = new HashMap();
paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$music"));
@@ -246,7 +255,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
- TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+ TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@@ -259,11 +268,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$literature", ir);
IOUtils.close(tr, ir, iw, tw);
- IOUtils.close(dirs[0]);
- }
-
- private Directory[][] getDirs() {
- return FacetTestUtils.createIndexTaxonomyDirs(1);
+ IOUtils.close(indexDir, taxoDir);
}
private void assertCorrectResults(FacetsCollector facetsCollector) throws IOException {
@@ -274,7 +279,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
Iterable subResults = resNode.subResults;
Iterator subIter = subResults.iterator();
- checkResult(resNode, "Band", 5.0);
checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Punk", 1.0);
@@ -283,7 +287,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
subResults = resNode.subResults;
subIter = subResults.iterator();
- checkResult(resNode, "Band", 5.0);
checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
@@ -297,7 +300,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
subResults = resNode.subResults;
subIter = subResults.iterator();
- checkResult(resNode, "Author", 3.0);
checkResult(subIter.next(), "Author/Kurt Vonnegut", 1.0);
checkResult(subIter.next(), "Author/Stephen King", 1.0);
checkResult(subIter.next(), "Author/Mark Twain", 1.0);
@@ -307,7 +309,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
subResults = resNode.subResults;
subIter = subResults.iterator();
- checkResult(resNode, "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/U2", 1.0);
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java
index a63e6108e00..8575d32ca08 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search;
import java.util.List;
import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -31,18 +32,21 @@ import org.junit.Before;
public class TestSameRequestAccumulation extends FacetTestBase {
+ private FacetIndexingParams fip;
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
- initIndex();
+ fip = getFacetIndexingParams(Integer.MAX_VALUE);
+ initIndex(fip);
}
// Following LUCENE-4461 - ensure requesting the (exact) same request more
// than once does not alter the results
public void testTwoSameRequests() throws Exception {
final CountFacetRequest facetRequest = new CountFacetRequest(new CategoryPath("root"), 10);
- FacetSearchParams fsp = new FacetSearchParams(facetRequest);
+ FacetSearchParams fsp = new FacetSearchParams(fip, facetRequest);
FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
searcher.search(new MatchAllDocsQuery(), fc);
@@ -50,7 +54,7 @@ public class TestSameRequestAccumulation extends FacetTestBase {
final String expected = fc.getFacetResults().get(0).toString();
// now add the same facet request with duplicates (same instance and same one)
- fsp = new FacetSearchParams(facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
+ fsp = new FacetSearchParams(fip, facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
// make sure the search params holds 3 requests now
assertEquals(3, fsp.facetRequests.size());
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java
index bac10b19659..03d099e5210 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java
@@ -5,6 +5,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.params.ScoreFacetRequest;
@@ -37,11 +38,14 @@ import org.junit.Test;
/** Test ScoredDocIdCollector. */
public class TestScoredDocIdCollector extends FacetTestBase {
+ private FacetIndexingParams fip;
+
@Override
@Before
public void setUp() throws Exception {
super.setUp();
- initIndex();
+ fip = getFacetIndexingParams(Integer.MAX_VALUE);
+ initIndex(fip);
}
@Override
@@ -73,8 +77,8 @@ public class TestScoredDocIdCollector extends FacetTestBase {
// verify by facet values
CategoryPath cp = new CategoryPath("root","a");
- FacetSearchParams countFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(cp, 10));
- FacetSearchParams scoreFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new ScoreFacetRequest(cp, 10));
+ FacetSearchParams countFSP = new FacetSearchParams(fip, new CountFacetRequest(cp, 10));
+ FacetSearchParams scoreFSP = new FacetSearchParams(fip, new ScoreFacetRequest(cp, 10));
List countRes = findFacets(scoredDocIDs, countFSP);
List scoreRes = findFacets(scoredDocIDs, scoreFSP);
@@ -101,10 +105,8 @@ public class TestScoredDocIdCollector extends FacetTestBase {
}
// compute facets with certain facet requests and docs
- private List findFacets(ScoredDocIDs sDocids,
- FacetSearchParams facetSearchParams) throws IOException {
- FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(
- facetSearchParams, indexReader, taxoReader);
+ private List findFacets(ScoredDocIDs sDocids, FacetSearchParams facetSearchParams) throws IOException {
+ FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(facetSearchParams, indexReader, taxoReader);
List res = fAccumulator.accumulate(sDocids);
// Results are ready, printing them...
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
index 06e09480458..9bc3dc1cc0e 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
@@ -113,7 +113,6 @@ public class TestStandardFacetsAccumulator extends FacetTestCase {
List results = fc.getFacetResults();
assertEquals("received too many facet results", 1, results.size());
FacetResultNode frn = results.get(0).getFacetResultNode();
- assertEquals("wrong weight for \"A\"", 4, (int) frn.value);
assertEquals("wrong number of children", 2, frn.subResults.size());
for (FacetResultNode node : frn.subResults) {
assertEquals("wrong weight for child " + node.label, 2, (int) node.value);
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
index 856e36f94a4..c5621cb9f64 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
@@ -181,7 +181,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
boolean hasDoctor = "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(9, fr.getNumValidDescendants());
FacetResultNode parentRes = fr.getFacetResultNode();
- assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values: a/b with 8 and a/c with 6
// a/b has two children a/b/2 with value 3, and a/b/1 with value 2.
@@ -217,7 +216,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(9, fr.getNumValidDescendants());
parentRes = fr.getFacetResultNode();
- assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values: a/b with 8 and a/c with 6
// a/b has two children a/b/2 with value 3, and a/b/1 with value 2.
@@ -234,7 +232,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(4, fr.getNumValidDescendants(), 4);
parentRes = fr.getFacetResultNode();
- assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values:
// a/b with value 8 and a/c with value 6
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
index a619529e7e5..a82a19c6559 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
@@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest.ResultMode;
@@ -73,7 +75,9 @@ public class TestTopKResultsHandler extends BaseTestTopK {
@Test
public void testSimple() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ OrdinalPolicy op = fip.getCategoryListParams(null).getOrdinalPolicy(null);
+ initIndex(fip);
List facetRequests = new ArrayList();
facetRequests.add(new CountFacetRequest(new CategoryPath("a"), 100));
@@ -87,8 +91,8 @@ public class TestTopKResultsHandler extends BaseTestTopK {
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "c"), 100));
// do different facet counts and compare to control
- FacetSearchParams sParams = getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
-
+ FacetSearchParams sParams = getFacetSearchParams(facetRequests, fip);
+
FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
@Override
protected FacetsAccumulator initFacetsAccumulator(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
@@ -100,17 +104,21 @@ public class TestTopKResultsHandler extends BaseTestTopK {
searcher.search(new MatchAllDocsQuery(), fc);
List facetResults = fc.getFacetResults();
-
+
FacetResult fr = facetResults.get(0);
FacetResultNode parentRes = fr.getFacetResultNode();
- assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ }
FacetResultNode[] frn = resultNodesAsArray(parentRes);
assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
fr = facetResults.get(1);
parentRes = fr.getFacetResultNode();
- assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
@@ -121,7 +129,9 @@ public class TestTopKResultsHandler extends BaseTestTopK {
fr = facetResults.get(2);
parentRes = fr.getFacetResultNode();
- assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(2.0, frn[0].value, Double.MIN_VALUE);
assertEquals(2.0, frn[1].value, Double.MIN_VALUE);
@@ -130,13 +140,17 @@ public class TestTopKResultsHandler extends BaseTestTopK {
fr = facetResults.get(3);
parentRes = fr.getFacetResultNode();
- assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(0, frn.length);
fr = facetResults.get(4);
parentRes = fr.getFacetResultNode();
- assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+ if (op == OrdinalPolicy.ALL_PARENTS) {
+ assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+ }
frn = resultNodesAsArray(parentRes);
assertEquals(1.0, frn[0].value, Double.MIN_VALUE);
closeAll();
@@ -149,12 +163,12 @@ public class TestTopKResultsHandler extends BaseTestTopK {
@Test
public void testGetMaxIntFacets() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ initIndex(fip);
// do different facet counts and compare to control
CategoryPath path = new CategoryPath("a", "b");
- FacetSearchParams sParams = getFacetSearchParams(getFacetIndexingParams(partitionSize),
- new CountFacetRequest(path, Integer.MAX_VALUE));
+ FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, Integer.MAX_VALUE));
FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
@Override
@@ -174,7 +188,7 @@ public class TestTopKResultsHandler extends BaseTestTopK {
// As a control base results, ask for top-1000 results
FacetSearchParams sParams2 = getFacetSearchParams(
- getFacetIndexingParams(partitionSize), new CountFacetRequest(path, Integer.MAX_VALUE));
+ fip, new CountFacetRequest(path, Integer.MAX_VALUE));
FacetsCollector fc2 = new StandardFacetsCollector(sParams2, indexReader, taxoReader) {
@Override
@@ -207,12 +221,11 @@ public class TestTopKResultsHandler extends BaseTestTopK {
@Test
public void testSimpleSearchForNonexistentFacet() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ initIndex(fip);
CategoryPath path = new CategoryPath("Miau Hattulla");
- FacetSearchParams sParams = getFacetSearchParams(
- getFacetIndexingParams(partitionSize),
- new CountFacetRequest(path, 10));
+ FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, 10));
FacetsCollector fc = FacetsCollector.create(sParams, indexReader, taxoReader);
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
index 25094f0af5e..083f52306f1 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
@@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
@@ -32,10 +33,10 @@ import org.junit.Test;
public class TestTopKResultsHandlerRandom extends BaseTestTopK {
- private List countFacets(int partitionSize, int numResults, final boolean doComplement)
+ private List countFacets(FacetIndexingParams fip, int numResults, final boolean doComplement)
throws IOException {
Query q = new MatchAllDocsQuery();
- FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, partitionSize);
+ FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, fip);
FacetsCollector fc = new StandardFacetsCollector(facetSearchParams, indexReader, taxoReader) {
@Override
protected FacetsAccumulator initFacetsAccumulator(
@@ -59,7 +60,8 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
@Test
public void testTopCountsOrder() throws Exception {
for (int partitionSize : partitionSizes) {
- initIndex(partitionSize);
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+ initIndex(fip);
/*
* Try out faceted search in it's most basic form (no sampling nor complement
@@ -67,7 +69,7 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
* being indexed, and later on an "over-all" faceted search is performed. The
* results are checked against the DF of each facet by itself
*/
- List facetResults = countFacets(partitionSize, 100000, false);
+ List facetResults = countFacets(fip, 100000, false);
assertCountsAndCardinality(facetCountsTruth(), facetResults);
/*
@@ -77,10 +79,10 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
* place in here. The results are checked against the a regular (a.k.a
* no-complement, no-sampling) faceted search with the same parameters.
*/
- facetResults = countFacets(partitionSize, 100000, true);
+ facetResults = countFacets(fip, 100000, true);
assertCountsAndCardinality(facetCountsTruth(), facetResults);
- List allFacetResults = countFacets(partitionSize, 100000, false);
+ List allFacetResults = countFacets(fip, 100000, false);
HashMap all = new HashMap();
int maxNumNodes = 0;
@@ -108,7 +110,7 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
if (VERBOSE) {
System.out.println("------- verify for "+n+" top results");
}
- List someResults = countFacets(partitionSize, n, false);
+ List someResults = countFacets(fip, n, false);
k = 0;
for (FacetResult fr : someResults) {
FacetResultNode topResNode = fr.getFacetResultNode();
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
index 82388b7fabb..f84678bb46b 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
@@ -3,12 +3,20 @@ package org.apache.lucene.facet.search;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
+import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.FacetTestUtils;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
+import org.apache.lucene.facet.index.FacetFields;
+import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util._TestUtil;
@@ -33,9 +41,9 @@ import org.junit.Test;
public class TestTotalFacetCounts extends FacetTestCase {
- private static void initCache(int numEntries) {
+ private static void initCache() {
TotalFacetCountsCache.getSingleton().clear();
- TotalFacetCountsCache.getSingleton().setCacheSize(numEntries); // Set to keep one in mem
+ TotalFacetCountsCache.getSingleton().setCacheSize(1); // Set to keep one in mem
}
@Test
@@ -48,51 +56,60 @@ public class TestTotalFacetCounts extends FacetTestCase {
}
private void doTestWriteRead(final int partitionSize) throws IOException {
- initCache(1);
+ initCache();
- // Create temporary RAMDirectories
- Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
- // Create our index/taxonomy writers
- IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+ IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+ TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+
FacetIndexingParams iParams = new FacetIndexingParams() {
@Override
public int getPartitionSize() {
return partitionSize;
}
+
+ @Override
+ public CategoryListParams getCategoryListParams(CategoryPath category) {
+ return new CategoryListParams() {
+ @Override
+ public OrdinalPolicy getOrdinalPolicy(String dimension) {
+ return OrdinalPolicy.ALL_PARENTS;
+ }
+ };
+ }
};
// The counts that the TotalFacetCountsArray should have after adding
// the below facets to the index.
int[] expectedCounts = new int[] { 0, 3, 1, 3, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1 };
-
- // Add a facet to the index
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "e");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "d");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "g");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "z");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "a");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "1", "2");
- TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "c");
+ String[] categories = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };
+
+ FacetFields facetFields = new FacetFields(taxoWriter, iParams);
+ for (String cat : categories) {
+ Document doc = new Document();
+ facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
+ indexWriter.addDocument(doc);
+ }
// Commit Changes
- writers[0].close();
+ IOUtils.close(indexWriter, taxoWriter);
- IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+ DirectoryReader indexReader = DirectoryReader.open(indexDir);
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
int[] intArray = new int[iParams.getPartitionSize()];
TotalFacetCountsCache tfcc = TotalFacetCountsCache.getSingleton();
File tmpFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
- tfcc.store(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
+ tfcc.store(tmpFile, indexReader, taxoReader, iParams);
tfcc.clear(); // not really required because TFCC overrides on load(), but in the test we need not rely on this.
- tfcc.load(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
+ tfcc.load(tmpFile, indexReader, taxoReader, iParams);
// now retrieve the one just loaded
- TotalFacetCounts totalCounts = tfcc.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ TotalFacetCounts totalCounts = tfcc.getTotalCounts(indexReader, taxoReader, iParams);
int partition = 0;
- for (int i=0; i getResults() {
+ return results;
}
@Override
public void run() {
try {
- results = MultiCLSearcher.searchWithFacets(indexReader, taxoReader, iParams);
+ FacetSearchParams fsp = new FacetSearchParams(iParams, new CountFacetRequest(new CategoryPath("a"), 10),
+ new CountFacetRequest(new CategoryPath("b"), 10));
+ IndexSearcher searcher = new IndexSearcher(indexReader);
+ FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
+ searcher.search(new MatchAllDocsQuery(), fc);
+ results = fc.getFacetResults();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
- // Instantiate threads, but do not start them
Multi[] multis = new Multi[numThreads];
- for (int i = 0; i < numThreads - 1; i++) {
- multis[i] = new Multi(slowIndexReader, slowTaxoReader, MultiCLIndexer.MULTI_IPARAMS);
+ for (int i = 0; i < numThreads; i++) {
+ multis[i] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
}
- // The last thread uses ONLY the DefaultFacetIndexingParams so that
- // it references a different TFC cache. This will still result
- // in valid results, but will only search one of the category lists
- // instead of all of them.
- multis[numThreads - 1] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
- // Gentleman, start your engines
for (Multi m : multis) {
m.start();
}
// Wait for threads and get results
- ExampleResult[] multiResults = new ExampleResult[numThreads];
- for (int i = 0; i < numThreads; i++) {
- multis[i].join();
- multiResults[i] = multis[i].getResults();
+ String[] expLabelsA = new String[] { "a/d", "a/e", "a/b" };
+ String[] expLabelsB = new String[] { "b/c", "b/a" };
+ for (Multi m : multis) {
+ m.join();
+ List facetResults = m.getResults();
+ assertEquals("expected two results", 2, facetResults.size());
+
+ FacetResultNode nodeA = facetResults.get(0).getFacetResultNode();
+ int i = 0;
+ for (FacetResultNode node : nodeA.subResults) {
+ assertEquals("wrong count", 1, (int) node.value);
+ assertEquals(expLabelsA[i++], node.label.toString('/'));
+ }
+
+ FacetResultNode nodeB = facetResults.get(1).getFacetResultNode();
+ i = 0;
+ for (FacetResultNode node : nodeB.subResults) {
+ assertEquals("wrong count", 1, (int) node.value);
+ assertEquals(expLabelsB[i++], node.label.toString('/'));
+ }
}
-
- // Each of the (numThreads-1) should have the same predictable
- // results, which we test for here.
- for (int i = 0; i < numThreads - 1; i++) {
- ExampleResult eResults = multiResults[i];
- TestMultiCLExample.assertCorrectMultiResults(eResults);
- }
-
- // The last thread, which only searched over the
- // DefaultFacetIndexingParams,
- // has its own results
- ExampleResult eResults = multiResults[numThreads - 1];
- List results = eResults.getFacetResults();
- assertEquals(3, results.size());
- String[] expLabels = new String[] { "5", "5/5", "6/2" };
- double[] expValues = new double[] { 0.0, 0.0, 1.0 };
- for (int i = 0; i < 3; i++) {
- FacetResult result = results.get(i);
- assertNotNull("Result should not be null", result);
- FacetResultNode resNode = result.getFacetResultNode();
- assertEquals("Invalid label", expLabels[i], resNode.label.toString());
- assertEquals("Invalid value", expValues[i], resNode.value, 0.0);
- assertEquals("Invalid number of subresults", 0, resNode.subResults.size());
- }
- // we're done, close the index reader and the taxonomy.
- slowIndexReader.close();
- slowTaxoReader.close();
- indexDir.close();
- taxoDir.close();
+
+ IOUtils.close(slowIndexReader, slowTaxoReader, indexDir, taxoDir);
}
/**
@@ -245,77 +225,78 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
@Test
public void testGenerationalConsistency() throws Exception {
// Create temporary RAMDirectories
- Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
// Create our index/taxonomy writers
- IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+ IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+ TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
// Add a facet to the index
- addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+ addFacets(iParams, indexWriter, taxoWriter, "a", "b");
// Commit Changes
- writers[0].indexWriter.commit();
- writers[0].taxWriter.commit();
+ indexWriter.commit();
+ taxoWriter.commit();
// Open readers
- IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+ DirectoryReader indexReader = DirectoryReader.open(indexDir);
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
// As this is the first time we have invoked the TotalFacetCountsManager,
// we should expect to compute and not read from disk.
- TotalFacetCounts totalCounts =
- TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
int prevGen = assertRecomputed(totalCounts, 0, "after first attempt to get it!");
// Repeating same operation should pull from the cache - not recomputed.
assertTrue("Should be obtained from cache at 2nd attempt",totalCounts ==
- TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+ TFC.getTotalCounts(indexReader, taxoReader, iParams));
// Repeat the same operation as above. but clear first - now should recompute again
initCache();
- totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 3rd attempt to get it!");
//store to file
File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
initCache();
- TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
- totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ TFC.store(outputFile, indexReader, taxoReader, iParams);
+ totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 4th attempt to get it!");
//clear and load
initCache();
- TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
- totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ TFC.load(outputFile, indexReader, taxoReader, iParams);
+ totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertReadFromDisc(totalCounts, prevGen, "after 5th attempt to get it!");
// Add a new facet to the index, commit and refresh readers
- addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
- writers[0].indexWriter.close();
- writers[0].taxWriter.close();
+ addFacets(iParams, indexWriter, taxoWriter, "c", "d");
+ IOUtils.close(indexWriter, taxoWriter);
- DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
+ TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
assertNotNull(newTaxoReader);
- assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > readers[0].taxReader.getSize());
- readers[0].taxReader.close();
- readers[0].taxReader = newTaxoReader;
+ assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > taxoReader.getSize());
+ taxoReader.close();
+ taxoReader = newTaxoReader;
- DirectoryReader r2 = DirectoryReader.openIfChanged(readers[0].indexReader);
+ DirectoryReader r2 = DirectoryReader.openIfChanged(indexReader);
assertNotNull(r2);
- readers[0].indexReader.close();
- readers[0].indexReader = r2;
+ indexReader.close();
+ indexReader = r2;
// now use the new reader - should recompute
- totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertRecomputed(totalCounts, prevGen, "after updating the index - 7th attempt!");
// try again - should not recompute
assertTrue("Should be obtained from cache at 8th attempt",totalCounts ==
- TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+ TFC.getTotalCounts(indexReader, taxoReader, iParams));
- readers[0].close();
+ IOUtils.close(indexReader, taxoReader);
outputFile.delete();
- IOUtils.close(dirs[0]);
+ IOUtils.close(indexDir, taxoDir);
}
private int assertReadFromDisc(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
@@ -341,10 +322,12 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
*/
@Test
public void testGrowingTaxonomy() throws Exception {
- // Create temporary RAMDirectories
- Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+ Directory indexDir = newDirectory();
+ Directory taxoDir = newDirectory();
+
// Create our index/taxonomy writers
- IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+ IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+ TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
FacetIndexingParams iParams = new FacetIndexingParams() {
@Override
public int getPartitionSize() {
@@ -352,37 +335,38 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
}
};
// Add a facet to the index
- addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+ addFacets(iParams, indexWriter, taxoWriter, "a", "b");
// Commit Changes
- writers[0].indexWriter.commit();
- writers[0].taxWriter.commit();
+ indexWriter.commit();
+ taxoWriter.commit();
- IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+ DirectoryReader indexReader = DirectoryReader.open(indexDir);
+ TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
// Create TFC and write cache to disk
File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
- TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+ TFC.store(outputFile, indexReader, taxoReader, iParams);
// Make the taxonomy grow without touching the index
for (int i = 0; i < 10; i++) {
- writers[0].taxWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
+ taxoWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
}
- writers[0].taxWriter.commit();
- DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
+ taxoWriter.commit();
+ TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
assertNotNull(newTaxoReader);
- readers[0].taxReader.close();
- readers[0].taxReader = newTaxoReader;
+ taxoReader.close();
+ taxoReader = newTaxoReader;
initCache();
// With the bug, this next call should result in an exception
- TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
- TotalFacetCounts totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ TFC.load(outputFile, indexReader, taxoReader, iParams);
+ TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
assertReadFromDisc(totalCounts, 0, "after reading from disk.");
+
outputFile.delete();
- writers[0].close();
- readers[0].close();
- IOUtils.close(dirs[0]);
+ IOUtils.close(indexWriter, taxoWriter, indexReader, taxoReader);
+ IOUtils.close(indexDir, taxoDir);
}
/**
@@ -445,46 +429,52 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
*/
@Test
public void testMultipleIndices() throws IOException {
- // Create temporary RAMDirectories
- Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(2);
+ Directory indexDir1 = newDirectory(), indexDir2 = newDirectory();
+ Directory taxoDir1 = newDirectory(), taxoDir2 = newDirectory();
+
// Create our index/taxonomy writers
- IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+ IndexWriter indexWriter1 = new IndexWriter(indexDir1, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+ IndexWriter indexWriter2 = new IndexWriter(indexDir2, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+ TaxonomyWriter taxoWriter1 = new DirectoryTaxonomyWriter(taxoDir1);
+ TaxonomyWriter taxoWriter2 = new DirectoryTaxonomyWriter(taxoDir2);
FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
// Add a facet to the index
- addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
- addFacets(iParams, writers[1].indexWriter, writers[1].taxWriter, "d", "e");
+ addFacets(iParams, indexWriter1, taxoWriter1, "a", "b");
+ addFacets(iParams, indexWriter1, taxoWriter1, "d", "e");
// Commit Changes
- writers[0].indexWriter.commit();
- writers[0].taxWriter.commit();
- writers[1].indexWriter.commit();
- writers[1].taxWriter.commit();
+ indexWriter1.commit();
+ indexWriter2.commit();
+ taxoWriter1.commit();
+ taxoWriter2.commit();
// Open two readers
- IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+ DirectoryReader indexReader1 = DirectoryReader.open(indexDir1);
+ DirectoryReader indexReader2 = DirectoryReader.open(indexDir2);
+ TaxonomyReader taxoReader1 = new DirectoryTaxonomyReader(taxoDir1);
+ TaxonomyReader taxoReader2 = new DirectoryTaxonomyReader(taxoDir2);
// As this is the first time we have invoked the TotalFacetCountsManager, we
// should expect to compute.
- TotalFacetCounts totalCounts0 =
- TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ TotalFacetCounts totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
int prevGen = -1;
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 1");
assertTrue("attempt 1b for same input [0] shout find it in cache",
- totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+ totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
// 2nd Reader - As this is the first time we have invoked the
// TotalFacetCountsManager, we should expect a state of NEW to be returned.
- TotalFacetCounts totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+ TotalFacetCounts totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 2");
assertTrue("attempt 2b for same input [1] shout find it in cache",
- totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
+ totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
// Right now cache size is one, so first TFC is gone and should be recomputed
- totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 3");
// Similarly will recompute the second result
- totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+ totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 4");
// Now we set the cache size to two, meaning both should exist in the
@@ -492,23 +482,19 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
TFC.setCacheSize(2);
// Re-compute totalCounts0 (was evicted from the cache when the cache was smaller)
- totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+ totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 5");
// now both are in the larger cache and should not be recomputed
- totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+ totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
assertTrue("with cache of size 2 res no. 0 should come from cache",
- totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+ totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
assertTrue("with cache of size 2 res no. 1 should come from cache",
- totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
+ totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
- writers[0].close();
- writers[1].close();
- readers[0].close();
- readers[1].close();
- for (Directory[] dirset : dirs) {
- IOUtils.close(dirset);
- }
+ IOUtils.close(indexWriter1, indexWriter2, taxoWriter1, taxoWriter2);
+ IOUtils.close(indexReader1, indexReader2, taxoReader1, taxoReader2);
+ IOUtils.close(indexDir1, indexDir2, taxoDir1, taxoDir2);
}
}
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
index d7de185c0b6..aca59444346 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search.sampling;
import java.util.List;
import java.util.Random;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.BaseTestTopK;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.FacetsCollector;
@@ -46,8 +47,8 @@ public abstract class BaseSampleTestTopK extends BaseTestTopK {
protected static final int RETRIES = 10;
@Override
- protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
- FacetSearchParams res = super.searchParamsWithRequests(numResults, partitionSize);
+ protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
+ FacetSearchParams res = super.searchParamsWithRequests(numResults, fip);
for (FacetRequest req : res.facetRequests) {
// randomize the way we aggregate results
if (random().nextBoolean()) {
@@ -71,20 +72,23 @@ public abstract class BaseSampleTestTopK extends BaseTestTopK {
boolean useRandomSampler = random().nextBoolean();
for (int partitionSize : partitionSizes) {
try {
- initIndex(partitionSize);
+ // complements return counts for all ordinals, so force ALL_PARENTS indexing
+ // so that it's easier to compare
+ FacetIndexingParams fip = getFacetIndexingParams(partitionSize, true);
+ initIndex(fip);
// Get all of the documents and run the query, then do different
// facet counts and compare to control
Query q = new TermQuery(new Term(CONTENT_FIELD, BETA)); // 90% of the docs
ScoredDocIdCollector docCollector = ScoredDocIdCollector.create(indexReader.maxDoc(), false);
- FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, partitionSize);
+ FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, fip);
FacetsCollector fc = FacetsCollector.create(expectedSearchParams, indexReader, taxoReader);
searcher.search(q, MultiCollector.wrap(docCollector, fc));
List expectedResults = fc.getFacetResults();
- FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, partitionSize);
+ FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, fip);
// try several times in case of failure, because the test has a chance to fail
// if the top K facets are not sufficiently common with the sample set
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java
index 1117fcf1d73..71b7058035d 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java
@@ -3,7 +3,6 @@ package org.apache.lucene.facet.search.sampling;
import java.io.IOException;
import java.util.Collections;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
@@ -95,7 +94,7 @@ public class OversampleWithDepthTest extends FacetTestCase {
}
private void index100Docs(Directory indexDir, Directory taxoDir, FacetIndexingParams fip) throws IOException {
- IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer());
+ IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
IndexWriter w = new IndexWriter(indexDir, iwc);
TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
diff --git a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
index 2ae22a13c1c..3533e64118d 100644
--- a/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
+++ b/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
@@ -3,7 +3,7 @@ package org.apache.lucene.facet.taxonomy.directory;
import java.io.IOException;
import java.util.Random;
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@@ -256,7 +256,7 @@ public class TestDirectoryTaxonomyReader extends FacetTestCase {
// hold onto IW to forceMerge
// note how we don't close it, since DTW will close it.
final IndexWriter iw = new IndexWriter(dir,
- new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(new LogByteSizeMergePolicy()));
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
@Override
@@ -299,7 +299,7 @@ public class TestDirectoryTaxonomyReader extends FacetTestCase {
// hold onto IW to forceMerge
// note how we don't close it, since DTW will close it.
final IndexWriter iw = new IndexWriter(dir,
- new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
+ new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(new LogByteSizeMergePolicy()));
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
@Override
diff --git a/lucene/site/xsl/index.xsl b/lucene/site/xsl/index.xsl
index a69b698ef57..a5e0ec477aa 100644
--- a/lucene/site/xsl/index.xsl
+++ b/lucene/site/xsl/index.xsl
@@ -58,7 +58,8 @@
Introduction to Lucene's APIs:
High-level summary of the different Lucene packages.
Analysis overview:
- Introduction to Lucene's analysis API.
+ Introduction to Lucene's analysis API. See also the
+ TokenStream consumer workflow.
Reference Documents
diff --git a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
index 9182babda19..fc620281b5c 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
@@ -19,6 +19,7 @@ package org.apache.lucene.analysis;
import java.io.IOException;
import java.io.Reader;
+import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -89,8 +90,8 @@ public class MockCharFilter extends CharFilter {
@Override
public int correct(int currentOff) {
- SortedMap subMap = corrections.subMap(0, currentOff+1);
- int ret = subMap.isEmpty() ? currentOff : currentOff + subMap.get(subMap.lastKey());
+ Map.Entry lastEntry = corrections.lowerEntry(currentOff+1);
+ int ret = lastEntry == null ? currentOff : currentOff + lastEntry.getValue();
assert ret >= 0 : "currentOff=" + currentOff + ",diff=" + (ret-currentOff);
return ret;
}
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 45e099a3a4a..c88e3a238dd 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -95,6 +95,11 @@ Bug Fixes
* SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
+* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
+
+* SOLR-4380: Replicate after startup option would not replicate until the
+ IndexWriter was lazily opened. (Mark Miller, Gregg Donovan)
+
Optimizations
----------------------
@@ -115,6 +120,8 @@ Optimizations
* SOLR-4284: Admin UI - make core list scrollable separate from the rest of
the UI (steffkes)
+* SOLR-4364: Admin UI - Locale based number formatting (steffkes)
+
Other Changes
----------------------
@@ -123,6 +130,8 @@ Other Changes
* SOLR-4353: Renamed example jetty context file to reduce confusion (hossman)
+* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
+
================== 4.1.0 ==================
Versions of Major Components
diff --git a/solr/build.xml b/solr/build.xml
index 7110d6ad89a..9c0cfdf6fb1 100644
--- a/solr/build.xml
+++ b/solr/build.xml
@@ -560,17 +560,8 @@
-
-
-
-
-
-
-
-
-
-
-
+
+