Merged /lucene/dev/trunk:r1439980-1440837

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1440839 13f79535-47bb-0310-9956-ffa450edef68

dev-tools/idea/lucene/demo/demo.iml

@@ -24,6 +24,7 @@
     </orderEntry>
     <orderEntry type="module" module-name="analysis-common" />
     <orderEntry type="module" module-name="lucene-core" />
+    <orderEntry type="module" module-name="facet" />
     <orderEntry type="module" module-name="queryparser" />
   </component>
 </module>

dev-tools/maven/lucene/demo/pom.xml.template

@@ -59,6 +59,11 @@
       <artifactId>lucene-analyzers-common</artifactId>
       <version>${project.version}</version>
     </dependency>
+    <dependency>
+      <groupId>${project.groupId}</groupId>
+      <artifactId>lucene-facet</artifactId>
+      <version>${project.version}</version>
+    </dependency>
     <dependency>
       <groupId>${project.groupId}</groupId>
       <artifactId>lucene-queryparser</artifactId>

lucene/CHANGES.txt

@@ -61,6 +61,11 @@ Optimizations
 * LUCENE-4690: Performance improvements and non-hashing versions
   of NumericUtils.*ToPrefixCoded() (yonik)
 
+* LUCENE-4715: CategoryListParams.getOrdinalPolicy now allows to return a
+  different OrdinalPolicy per dimension, to better tune how you index
+  facets. Also added OrdinalPolicy.ALL_BUT_DIMENSION. 
+  (Shai Erera, Michael McCandless)
+
 New Features
 
 * LUCENE-4686: New specialized DGapVInt8IntEncoder for facets (now the 
@@ -115,6 +120,9 @@ Bug Fixes
   you have a taxonomy index created with such strings, you should rebuild it.
   (Michael McCandless, Shai Erera)
 
+* LUCENE-4732: Fixed TermsEnum.seekCeil/seekExact on term vectors.
+  (Adrien Grand, Robert Muir)
+
 ======================= Lucene 4.1.0 =======================
 
 Changes in backwards compatibility policy

lucene/build.xml

@@ -191,8 +191,7 @@
         <exclude name="analysis/stempel/classes/java/org/egothor/stemmer/Compile.class"/>
         <exclude name="analysis/stempel/classes/java/org/egothor/stemmer/DiffIt.class"/>
         <exclude name="benchmark/**"/>
-        <exclude name="demo/classes/java/org/apache/lucene/demo/IndexFiles.class"/>
+        <exclude name="demo/classes/java/org/apache/lucene/**"/>
-        <exclude name="demo/classes/java/org/apache/lucene/demo/SearchFiles.class"/>
         <exclude name="misc/classes/java/org/apache/lucene/index/CompoundFileExtractor.class"/>
         <exclude name="misc/classes/java/org/apache/lucene/index/IndexSplitter.class"/>
         <exclude name="misc/classes/java/org/apache/lucene/index/MultiPassIndexSplitter.class"/>

lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java

@@ -832,10 +832,18 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem
         }
       }
       // linear scan
-      do {
+      while (true) {
-        next();
+        final BytesRef term = next();
-      } while (ord < numTerms - 1 && term().compareTo(text) < 0);
+        if (term == null) {
-      return term().equals(text) ? SeekStatus.FOUND : SeekStatus.END;
+          return SeekStatus.END;
+        }
+        final int cmp = term.compareTo(text);
+        if (cmp > 0) {
+          return SeekStatus.NOT_FOUND;
+        } else if (cmp == 0) {
+          return SeekStatus.FOUND;
+        }
+      }
     }
 
     @Override

lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java

@@ -422,7 +422,7 @@ public class Lucene40TermVectorsReader extends TermVectorsReader implements Clos
       this.storePayloads = storePayloads;
       nextTerm = 0;
       tvf.seek(tvfFPStart);
-      tvfFP = 1+tvfFPStart;
+      tvfFP = tvfFPStart;
       positions = null;
       startOffsets = null;
       endOffsets = null;

lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java

@@ -38,6 +38,7 @@ import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.FixedBitSet;
 import org.apache.lucene.util.LineFileDocs;
 import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
 import org.apache.lucene.util.automaton.AutomatonTestUtil;
 import org.apache.lucene.util.automaton.CompiledAutomaton;
 import org.apache.lucene.util.automaton.RegExp;
@@ -236,11 +237,11 @@ public class TestDuelingCodecs extends LuceneTestCase {
   private void assertTermsSeeking(Terms leftTerms, Terms rightTerms) throws Exception {
     TermsEnum leftEnum = null;
     TermsEnum rightEnum = null;
-    
+
     // just an upper bound
     int numTests = atLeast(20);
     Random random = random();
-    
+
     // collect this number of terms from the left side
     HashSet<BytesRef> tests = new HashSet<BytesRef>();
     int numPasses = 0;
@@ -264,36 +265,50 @@ public class TestDuelingCodecs extends LuceneTestCase {
           byte newbytes[] = new byte[term.length+5];
           System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length);
           tests.add(new BytesRef(newbytes, 5, term.length));
+        } else if (code == 3) {
+          switch (random().nextInt(3)) {
+            case 0:
+              tests.add(new BytesRef()); // before the first term
+              break;
+            case 1:
+              tests.add(new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF})); // past the last term
+              break;
+            case 2:
+              tests.add(new BytesRef(_TestUtil.randomSimpleString(random()))); // random term
+              break;
+            default:
+              throw new AssertionError();
+          }
         }
       }
       numPasses++;
     }
-    
+
+    rightEnum = rightTerms.iterator(rightEnum);
+
     ArrayList<BytesRef> shuffledTests = new ArrayList<BytesRef>(tests);
     Collections.shuffle(shuffledTests, random);
-    
+
     for (BytesRef b : shuffledTests) {
-      leftEnum = leftTerms.iterator(leftEnum);
+      if (rarely()) {
-      rightEnum = rightTerms.iterator(rightEnum);
+        // reuse the enums
-      
+        leftEnum = leftTerms.iterator(leftEnum);
-      assertEquals(info, leftEnum.seekExact(b, false), rightEnum.seekExact(b, false));
+        rightEnum = rightTerms.iterator(rightEnum);
-      assertEquals(info, leftEnum.seekExact(b, true), rightEnum.seekExact(b, true));
-      
-      SeekStatus leftStatus;
-      SeekStatus rightStatus;
-      
-      leftStatus = leftEnum.seekCeil(b, false);
-      rightStatus = rightEnum.seekCeil(b, false);
-      assertEquals(info, leftStatus, rightStatus);
-      if (leftStatus != SeekStatus.END) {
-        assertEquals(info, leftEnum.term(), rightEnum.term());
       }
-      
+
-      leftStatus = leftEnum.seekCeil(b, true);
+      final boolean useCache = random().nextBoolean();
-      rightStatus = rightEnum.seekCeil(b, true);
+      final boolean seekExact = random().nextBoolean();
-      assertEquals(info, leftStatus, rightStatus);
+
-      if (leftStatus != SeekStatus.END) {
+      if (seekExact) {
-        assertEquals(info, leftEnum.term(), rightEnum.term());
+        assertEquals(info, leftEnum.seekExact(b, useCache), rightEnum.seekExact(b, useCache));
+      } else {
+        SeekStatus leftStatus = leftEnum.seekCeil(b, useCache);
+        SeekStatus rightStatus = rightEnum.seekCeil(b, useCache);
+        assertEquals(info, leftStatus, rightStatus);
+        if (leftStatus != SeekStatus.END) {
+          assertEquals(info, leftEnum.term(), rightEnum.term());
+          assertTermStats(leftEnum, rightEnum);
+        }
       }
     }
   }

lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java

@@ -251,7 +251,7 @@ public class TestPayloadsOnVectors extends LuceneTestCase {
       for (int i = 0; i < len; ++i) {
         terms[i] = RandomPicks.randomFrom(random(), sampleTerms);
         if (weird) {
-          positionsIncrements[i] = random().nextInt(1 << 18);
+          positionsIncrements[i] = _TestUtil.nextInt(random(), 1, 1 << 18);
           startOffsets[i] = random().nextInt();
           endOffsets[i] = random().nextInt();
         } else if (i == 0) {

lucene/demo/build.xml

@@ -33,20 +33,22 @@
    <pathelement path="${analyzers-common.jar}"/>
    <pathelement path="${queryparser.jar}"/>
    <pathelement path="${lucene-core.jar}"/>
+   <pathelement path="${facet.jar}"/>
    <fileset dir="lib"/>
   </path>
 
-  <target name="javadocs" depends="javadocs-analyzers-common,javadocs-queryparser,compile-core">
+  <target name="javadocs" depends="javadocs-analyzers-common,javadocs-queryparser,javadocs-facet,compile-core">
     <!-- we link the example source in the javadocs, as its ref'ed elsewhere -->
     <invoke-module-javadoc linksource="yes">
       <links>
         <link href="../analyzers-common"/>
         <link href="../queryparser"/>
+      	<link href="../facet"/>
       </links>
     </invoke-module-javadoc>
   </target>
 
-  <target name="compile-core" depends="jar-analyzers-common,jar-queryparser,common.compile-core" />
+  <target name="compile-core" depends="jar-analyzers-common,jar-queryparser,jar-facet,common.compile-core" />
 
   <target name="default" depends="jar-core,build-web-demo"/>
 
@@ -63,6 +65,7 @@
       <lib file="${queries.jar}"/>
       <lib file="${queryparser.jar}"/>
       <lib file="${lucene-core.jar}"/>
+      <lib file="${facet.jar}"/>
     </war>
   </target>
   

lucene/demo/src/java/org/apache/lucene/demo/facet/ExampleResult.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example;
+package org.apache.lucene.demo.facet;
 
 import java.util.List;
 
@@ -29,18 +29,21 @@ import org.apache.lucene.facet.search.results.FacetResult;
  * @lucene.experimental
  */
 public class ExampleResult {
+  
+  /** Sole constructor. */
+  public ExampleResult() {}
 
   private List<FacetResult> facetResults;
 
   /**
-   * @return the facet results
+   * Returns the facet results
    */
   public List<FacetResult> getFacetResults() {
     return facetResults;
   }
 
   /**
-   * @param facetResults the facet results to set
+   * Sets the facet results
    */
   public void setFacetResults(List<FacetResult> facetResults) {
     this.facetResults = facetResults;

lucene/demo/src/java/org/apache/lucene/demo/facet/ExampleUtils.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example;
+package org.apache.lucene.demo.facet;
 
 import org.apache.lucene.util.Version;
 
@@ -20,15 +20,28 @@ import org.apache.lucene.util.Version;
  */
 
 /**
+ * Simple utility functions for the faceting examples
  * @lucene.experimental
  */
 public class ExampleUtils {
+  
+  /** No instance */
+  private ExampleUtils() {}
 
+  /** 
+   * True if the system property <code>tests.verbose</code> has been set.
+   * If true, it causes {@link #log(Object)} to print messages to the console.
+   */
   public static final boolean VERBOSE = Boolean.getBoolean("tests.verbose");
 
   /** The Lucene {@link Version} used by the example code. */
   public static final Version EXAMPLE_VER = Version.LUCENE_40;
   
+  /**
+   * Logs the String representation of <code>msg</code> to the console,
+   * if {@link #VERBOSE} is true. Otherwise, does nothing.
+   * @see #VERBOSE
+   */
   public static void log(Object msg) {
     if (VERBOSE) {
       System.out.println(msg.toString());

lucene/demo/src/java/org/apache/lucene/demo/facet/adaptive/AdaptiveMain.java

@@ -1,14 +1,14 @@
-package org.apache.lucene.facet.example.adaptive;
+package org.apache.lucene.demo.facet.adaptive;
 
 import java.util.List;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 
-import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.demo.facet.ExampleResult;
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
-import org.apache.lucene.facet.example.simple.SimpleIndexer;
+import org.apache.lucene.demo.facet.simple.SimpleIndexer;
-import org.apache.lucene.facet.example.simple.SimpleSearcher;
+import org.apache.lucene.demo.facet.simple.SimpleSearcher;
 import org.apache.lucene.facet.search.AdaptiveFacetsAccumulator;
 import org.apache.lucene.facet.search.results.FacetResult;
 
@@ -36,6 +36,9 @@ import org.apache.lucene.facet.search.results.FacetResult;
  * @lucene.experimental
  */
 public class AdaptiveMain {
+  
+  /** Sole constructor */
+  public AdaptiveMain() {}
 
   /**
    * Driver for the adaptive sample.
@@ -46,6 +49,7 @@ public class AdaptiveMain {
     ExampleUtils.log("DONE");
   }
 
+  /** Runs the adaptive sample and returns the facet results */
   public ExampleResult runSample() throws Exception {
 
     // create Directories for the search index and for the taxonomy index

lucene/demo/src/java/org/apache/lucene/demo/facet/adaptive/AdaptiveSearcher.java

@@ -1,9 +1,9 @@
-package org.apache.lucene.facet.example.adaptive;
+package org.apache.lucene.demo.facet.adaptive;
 
 import java.util.List;
 
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
-import org.apache.lucene.facet.example.simple.SimpleUtils;
+import org.apache.lucene.demo.facet.simple.SimpleUtils;
 import org.apache.lucene.facet.search.AdaptiveFacetsAccumulator;
 import org.apache.lucene.facet.search.ScoredDocIdCollector;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
@@ -46,6 +46,9 @@ import org.apache.lucene.store.Directory;
  */
 public class AdaptiveSearcher {
   
+  /** No instance */
+  private AdaptiveSearcher() {}
+  
   /**
    * Search with facets through the {@link AdaptiveFacetsAccumulator} 
    * @param indexDir Directory of the search index.

lucene/demo/src/java/org/apache/lucene/demo/facet/adaptive/package.html

@@ -0,0 +1,22 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html><head></head>
+<body>
+Facets example code for using AdaptiveFacetsAccumulator.
+</body>
+</html>

lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsIndexer.java

@@ -1,13 +1,13 @@
-package org.apache.lucene.facet.example.association;
+package org.apache.lucene.demo.facet.association;
 
+import org.apache.lucene.demo.facet.ExampleUtils;
+import org.apache.lucene.demo.facet.simple.SimpleUtils;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.facet.associations.AssociationsFacetFields;
 import org.apache.lucene.facet.associations.CategoryAssociation;
 import org.apache.lucene.facet.associations.CategoryAssociationsContainer;
-import org.apache.lucene.facet.example.ExampleUtils;
-import org.apache.lucene.facet.example.simple.SimpleUtils;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
@@ -40,6 +40,9 @@ import org.apache.lucene.store.Directory;
  * @lucene.experimental
  */
 public class CategoryAssociationsIndexer {
+  
+  /** No instance. */
+  private CategoryAssociationsIndexer() {}
 
   /**
    * Create an index, and adds to it sample documents and categories.

lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsMain.java

@@ -1,12 +1,12 @@
-package org.apache.lucene.facet.example.association;
+package org.apache.lucene.demo.facet.association;
 
 import java.util.List;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 
-import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.demo.facet.ExampleResult;
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
 import org.apache.lucene.facet.search.results.FacetResult;
 
 /*
@@ -33,6 +33,9 @@ import org.apache.lucene.facet.search.results.FacetResult;
  */
 public class CategoryAssociationsMain {
 
+  /** Sole constructor. */
+  public CategoryAssociationsMain() {}
+
   /**
    * Driver for the simple sample.
    * @throws Exception on error (no detailed exception handling here for sample simplicity
@@ -43,6 +46,9 @@ public class CategoryAssociationsMain {
     ExampleUtils.log("DONE");
   }
 
+  /**
+   * Runs the example demonstrating sum of int-association.
+   */
   public ExampleResult runSumIntAssociationSample() throws Exception {
 
     // create Directories for the search index and for the taxonomy index
@@ -61,6 +67,9 @@ public class CategoryAssociationsMain {
     return res;
   }
   
+  /**
+   * Runs the example demonstrating sum of float-association.
+   */
   public ExampleResult runSumFloatAssociationSample() throws Exception {
     
     // create Directories for the search index and for the taxonomy index

lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsSearcher.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example.association;
+package org.apache.lucene.demo.facet.association;
 
 import java.util.List;
 
@@ -6,7 +6,7 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.store.Directory;
 
-import org.apache.lucene.facet.example.simple.SimpleSearcher;
+import org.apache.lucene.demo.facet.simple.SimpleSearcher;
 import org.apache.lucene.facet.search.params.associations.AssociationFloatSumFacetRequest;
 import org.apache.lucene.facet.search.params.associations.AssociationIntSumFacetRequest;
 import org.apache.lucene.facet.search.results.FacetResult;
@@ -38,6 +38,9 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
  * @lucene.experimental
  */
 public class CategoryAssociationsSearcher {
+  
+  /** No instantiation */
+  private CategoryAssociationsSearcher() {}
 
   /** Search an index with a sum of int-association. */
   public static List<FacetResult> searchSumIntAssociation(Directory indexDir, Directory taxoDir) throws Exception {

lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsUtils.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example.association;
+package org.apache.lucene.demo.facet.association;
 
 import org.apache.lucene.facet.associations.CategoryAssociation;
 import org.apache.lucene.facet.associations.CategoryFloatAssociation;
@@ -23,9 +23,13 @@ import org.apache.lucene.facet.taxonomy.CategoryPath;
  */
 
 /**
+ * Categories for the facet examples
  * @lucene.experimental
  */
 public class CategoryAssociationsUtils {
+  
+  /** No instance */
+  private CategoryAssociationsUtils() {}
 
   /**
    * Categories: categories[D][N] == category-path with association no. N for
@@ -45,6 +49,9 @@ public class CategoryAssociationsUtils {
     }
   };
 
+  /**
+   * Associations (occurrences/confidence levels) for {@link #categories}
+   */
   public static CategoryAssociation[][] associations = {
     // Doc #1 associations
     {

lucene/demo/src/java/org/apache/lucene/demo/facet/association/package.html

@@ -0,0 +1,22 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html><head></head>
+<body>
+Facets example code for using associations.
+</body>
+</html>

lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/MultiCLIndexer.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example.multiCL;
+package org.apache.lucene.demo.facet.multiCL;
 
 import java.util.Arrays;
 import java.util.HashMap;
@@ -6,11 +6,11 @@ import java.util.List;
 import java.util.Map;
 import java.util.Random;
 
+import org.apache.lucene.demo.facet.ExampleUtils;
+import org.apache.lucene.demo.facet.simple.SimpleUtils;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.facet.example.ExampleUtils;
-import org.apache.lucene.facet.example.simple.SimpleUtils;
 import org.apache.lucene.facet.index.FacetFields;
 import org.apache.lucene.facet.index.params.CategoryListParams;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
@@ -48,14 +48,17 @@ import org.apache.lucene.store.RAMDirectory;
  * @lucene.experimental
  */
 public class MultiCLIndexer {
+  
+  /** No instance */
+  private MultiCLIndexer() {}
 
-  // Number of documents to index
+  /** Number of documents to index */
   public static int NUM_DOCS = 100;
-  // Number of facets to add per document
+  /** Number of facets to add per document */
   public static int NUM_FACETS_PER_DOC = 10;
-  // Number of tokens in title
+  /** Number of tokens in title */
   public static int TITLE_LENGTH = 5;
-  // Number of tokens in text
+  /** Number of tokens in text */
   public static int TEXT_LENGTH = 100;
   
   // Lorum ipsum to use as content - this will be tokenized and used for document
@@ -73,7 +76,7 @@ public class MultiCLIndexer {
       + "nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure"
       + "reprehenderit qui in ea voluptate velit esse quam nihil molestiae "
       + "consequatur vel illum qui dolorem eum fugiat quo voluptas nulla pariatur";
-  // PerDimensionIndexingParams for multiple category lists
+  /** PerDimensionIndexingParams for multiple category lists */
   public static final PerDimensionIndexingParams MULTI_IPARAMS;
 
   // Initialize PerDimensionIndexingParams
@@ -90,12 +93,16 @@ public class MultiCLIndexer {
   
   /**
    * Create an index, and adds to it sample documents and facets.
-   * @param indexDir Directory in which the index should be created.
+   * 
-   * @param taxoDir Directory in which the taxonomy index should be created.
+   * @param indexDir
-   * @throws Exception on error (no detailed exception handling here for sample simplicity
+   *          Directory in which the index should be created.
+   * @param taxoDir
+   *          Directory in which the taxonomy index should be created.
+   * @throws Exception
+   *           on error (no detailed exception handling here for sample
+   *           simplicity
    */
-  public static void index(Directory indexDir, Directory taxoDir)
+  public static void index(Directory indexDir, Directory taxoDir) throws Exception {
-      throws Exception {
 
     Random random = new Random(2003);
 
@@ -195,6 +202,7 @@ public class MultiCLIndexer {
         + nFacetsAdded + " facets.");
   }
 
+  /** Driver for the example */
   public static void main(String[] args) throws Exception {
     index(new RAMDirectory(), new RAMDirectory());
   }

lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/MultiCLMain.java

@@ -1,12 +1,12 @@
-package org.apache.lucene.facet.example.multiCL;
+package org.apache.lucene.demo.facet.multiCL;
 
 import java.util.List;
 
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 
-import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.demo.facet.ExampleResult;
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
 import org.apache.lucene.facet.search.results.FacetResult;
 
 /*
@@ -27,12 +27,17 @@ import org.apache.lucene.facet.search.results.FacetResult;
  */
 
 /**
+ * Driver for the multi sample.
+ * 
  * @lucene.experimental
  */
 public class MultiCLMain {
+  
+  /** Sole constructor. */
+  public MultiCLMain() {}
 
   /**
-   * Driver for the multi sample.
+   * Executes the multi sample.
    * 
    * @throws Exception
    *             on error (no detailed exception handling here for sample
@@ -43,6 +48,7 @@ public class MultiCLMain {
     ExampleUtils.log("DONE");
   }
 
+  /** Runs the multi sample and returns the facet results */
   public ExampleResult runSample() throws Exception {
 
     // create Directories for the search index and for the taxonomy index

lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/MultiCLSearcher.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example.multiCL;
+package org.apache.lucene.demo.facet.multiCL;
 
 import java.util.ArrayList;
 import java.util.List;
@@ -13,8 +13,8 @@ import org.apache.lucene.search.TopScoreDocCollector;
 import org.apache.lucene.store.Directory;
 
 import org.apache.lucene.search.MultiCollector;
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
-import org.apache.lucene.facet.example.simple.SimpleUtils;
+import org.apache.lucene.demo.facet.simple.SimpleUtils;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.FacetsCollector;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
@@ -49,6 +49,9 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
  * @lucene.experimental
  */
 public class MultiCLSearcher {
+  
+  /** No instance */
+  private MultiCLSearcher() {}
 
   /**
    * Search an index with facets.
@@ -78,6 +81,18 @@ public class MultiCLSearcher {
     return results;
   }
   
+  /**
+   * Search an index with facets.
+   * 
+   * @param indexReader
+   *            Reader over the search index.
+   * @param taxo
+   *            taxonomy reader.
+   * @throws Exception
+   *             on error (no detailed exception handling here for sample
+   *             simplicity
+   * @return facet results
+   */
   public static List<FacetResult> searchWithFacets(IndexReader indexReader,
       TaxonomyReader taxo, FacetIndexingParams iParams) throws Exception {
     // prepare searcher to search against

lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/package.html

@@ -0,0 +1,22 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html><head></head>
+<body>
+Facets example code for using multiple category lists.
+</body>
+</html>

lucene/demo/src/java/org/apache/lucene/demo/facet/package.html

@@ -0,0 +1,22 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html><head></head>
+<body>
+Facets example code.
+</body>
+</html>

lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleIndexer.java

@@ -1,12 +1,12 @@
-package org.apache.lucene.facet.example.simple;
+package org.apache.lucene.demo.facet.simple;
 
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.lucene.demo.facet.ExampleUtils;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
-import org.apache.lucene.facet.example.ExampleUtils;
 import org.apache.lucene.facet.index.FacetFields;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
@@ -39,6 +39,9 @@ import org.apache.lucene.store.Directory;
  * @lucene.experimental
  */
 public class SimpleIndexer {
+  
+  /** No instance */
+  private SimpleIndexer() {}
 
   /**
    * Create an index, and adds to it sample documents and facets.

lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleMain.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example.simple;
+package org.apache.lucene.demo.facet.simple;
 
 import java.util.List;
 
@@ -7,8 +7,8 @@ import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.RAMDirectory;
 
-import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.demo.facet.ExampleResult;
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.taxonomy.TaxonomyReader;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
@@ -36,6 +36,9 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
  * @lucene.experimental
  */
 public class SimpleMain {
+  
+  /** Sole constructor */
+  public SimpleMain() {}
 
   /**
    * Driver for the simple sample.
@@ -47,6 +50,7 @@ public class SimpleMain {
     ExampleUtils.log("DONE");
   }
 
+  /** Runs the simple sample and returns the facet results */
   public ExampleResult runSimple() throws Exception {
     // create Directories for the search index and for the taxonomy index
     Directory indexDir = new RAMDirectory();
@@ -72,6 +76,7 @@ public class SimpleMain {
     return res;
   }
 
+  /** Runs the simple sample and returns drilldown results */
   public ExampleResult runDrillDown() throws Exception {
     // create Directories for the search index and for the taxonomy index
     Directory indexDir = new RAMDirectory();

lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleSearcher.java

@@ -1,9 +1,9 @@
-package org.apache.lucene.facet.example.simple;
+package org.apache.lucene.demo.facet.simple;
 
 import java.util.Iterator;
 import java.util.List;
 
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.DrillDown;
 import org.apache.lucene.facet.search.FacetsCollector;
@@ -47,6 +47,9 @@ import org.apache.lucene.search.TopScoreDocCollector;
  */
 public class SimpleSearcher {
   
+  /** No instance */
+  private SimpleSearcher() {}
+  
   /**
    * Search an index with facets.
    * @param indexReader index reader.

lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleUtils.java

@@ -1,8 +1,8 @@
-package org.apache.lucene.facet.example.simple;
+package org.apache.lucene.demo.facet.simple;
 
 import org.apache.lucene.analysis.Analyzer;
 import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
-import org.apache.lucene.facet.example.ExampleUtils;
+import org.apache.lucene.demo.facet.ExampleUtils;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 
 /*
@@ -28,6 +28,9 @@ import org.apache.lucene.facet.taxonomy.CategoryPath;
  * @lucene.experimental
  */
 public class SimpleUtils {
+  
+  /** No instance */
+  private SimpleUtils() {}
 
   /** 
    * Documents text field.

lucene/demo/src/java/org/apache/lucene/demo/facet/simple/package.html

@@ -0,0 +1,22 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html><head></head>
+<body>
+Facets simple example code.
+</body>
+</html>

lucene/demo/src/test/org/apache/lucene/demo/facet/TestAdaptiveExample.java

@@ -1,10 +1,10 @@
-package org.apache.lucene.facet.example;
+package org.apache.lucene.demo.facet;
 
 import org.junit.Test;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.demo.facet.ExampleResult;
-import org.apache.lucene.facet.example.adaptive.AdaptiveMain;
+import org.apache.lucene.demo.facet.adaptive.AdaptiveMain;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more

lucene/demo/src/test/org/apache/lucene/demo/facet/TestAssociationExample.java

@@ -1,10 +1,10 @@
-package org.apache.lucene.facet.example;
+package org.apache.lucene.demo.facet;
 
 import org.junit.Test;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.demo.facet.ExampleResult;
-import org.apache.lucene.facet.example.association.CategoryAssociationsMain;
+import org.apache.lucene.demo.facet.association.CategoryAssociationsMain;
 import org.apache.lucene.facet.search.results.FacetResultNode;
 
 /*

lucene/demo/src/test/org/apache/lucene/demo/facet/TestMultiCLExample.java

@@ -1,4 +1,4 @@
-package org.apache.lucene.facet.example;
+package org.apache.lucene.demo.facet;
 
 import java.util.Iterator;
 import java.util.List;
@@ -6,7 +6,8 @@ import java.util.List;
 import org.junit.Test;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.example.multiCL.MultiCLMain;
+import org.apache.lucene.demo.facet.ExampleResult;
+import org.apache.lucene.demo.facet.multiCL.MultiCLMain;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.search.results.FacetResultNode;
 
@@ -45,7 +46,6 @@ public class TestMultiCLExample extends LuceneTestCase {
     assertNotNull("Result should not be null", result);
     FacetResultNode node = result.getFacetResultNode();
     assertEquals("Invalid label", "5", node.label.toString());
-    assertEquals("Invalid value", 2.0, node.value, 0.0);
     assertEquals("Invalid # of subresults", 3, node.subResults.size());
 
     Iterator<? extends FacetResultNode> subResults = node.subResults.iterator();

lucene/demo/src/test/org/apache/lucene/demo/facet/TestSimpleExample.java

@@ -1,12 +1,12 @@
-package org.apache.lucene.facet.example;
+package org.apache.lucene.demo.facet;
 
 import java.util.Iterator;
 
 import org.junit.Test;
 
 import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.facet.example.ExampleResult;
+import org.apache.lucene.demo.facet.ExampleResult;
-import org.apache.lucene.facet.example.simple.SimpleMain;
+import org.apache.lucene.demo.facet.simple.SimpleMain;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.search.results.FacetResultNode;
 

lucene/facet/build.xml

@@ -28,52 +28,6 @@
   
   <import file="../module-build.xml"/>
 
-  <property name="examples.dir" location="src/examples"/>
-
-  <path id="examples.classpath">
-    <path refid="classpath" />
-    <pathelement location="${build.dir}/classes/java" />
-    <pathelement path="${analyzers-common.jar}" />
-  </path>
-
-	<path id="test.classpath">
-    <path refid="test.base.classpath" />
-    <pathelement location="${build.dir}/classes/examples" />
-  </path>
-
-  <path id="classpath">
-    <!-- TODO, cut over tests to MockAnalyzer etc and nuke this dependency -->
-    <pathelement path="${analyzers-common.jar}" />
-    <path refid="base.classpath"/>
-  </path>
-
-  <target name="compile-examples" description="Compiles Facets examples">
-    <compile srcdir="${examples.dir}" destdir="${build.dir}/classes/examples">
-      <classpath refid="examples.classpath" />
-    </compile>
-  </target>
-  
-  <target name="jar-examples" depends="compile-examples">
-    <jarify basedir="${build.dir}/classes/examples" 
-            destfile="${build.dir}/${final.name}-examples.jar" 
-            title="Lucene Search Engine: ${ant.project.name}-examples"
-            manifest.file="${build.dir}/EXAMPLES-MANIFEST.MF">
-      <fileset dir="src/examples" />
-    </jarify>
-  </target>
-
-  <target name="compile-core" depends="jar-analyzers-common,common.compile-core,compile-examples" description="Compiles facet classes" />
-
-  <target name="jar-core" depends="common.jar-core,jar-examples" />
-
-  <target name="javadocs" depends="javadocs-analyzers-common,compile-core">
-    <invoke-module-javadoc>
-      <links>
-        <link href="../analyzers-common"/>
-      </links>
-    </invoke-module-javadoc>
-  </target>
-
   <target name="run-encoding-benchmark" depends="compile-test">
     <java classname="org.apache.lucene.util.encoding.EncodingSpeed" fork="true" failonerror="true">
       <classpath refid="test.classpath" />

lucene/facet/src/examples/org/apache/lucene/facet/example/simple/package.html

@@ -1,17 +0,0 @@
-<html>
-  <head>
-    <title>Simple faceted indexing and search sample</title>
-  </head>
-  <body>
-    <h1>Simple faceted indexing and search sample</h1>
-    
-    A simple faceted example, showing how to:
-    <ol>
-      <li>Create an index.</li>
-      <li>Add documents with facets to the index.</li>
-      <li>Search the index.</li>
-    </ol>  
-    
-    For more complex examples see the other sample code packages. 
-  </body>
-</html>

lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java

@@ -3,6 +3,7 @@ package org.apache.lucene.facet.index;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.HashMap;
+import java.util.Iterator;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -115,12 +116,12 @@ public class CountingListBuilder implements CategoryListBuilder {
   
   private final OrdinalsEncoder ordinalsEncoder;
   private final TaxonomyWriter taxoWriter;
-  private final OrdinalPolicy ordinalPolicy;
+  private final CategoryListParams clp;
   
   public CountingListBuilder(CategoryListParams categoryListParams, FacetIndexingParams indexingParams, 
       TaxonomyWriter taxoWriter) {
     this.taxoWriter = taxoWriter;
-    this.ordinalPolicy = categoryListParams.getOrdinalPolicy();
+    this.clp = categoryListParams;
     if (indexingParams.getPartitionSize() == Integer.MAX_VALUE) {
       ordinalsEncoder = new NoPartitionsOrdinalsEncoder(categoryListParams);
     } else {
@@ -141,16 +142,23 @@ public class CountingListBuilder implements CategoryListBuilder {
    */
   @Override
   public Map<String,BytesRef> build(IntsRef ordinals, Iterable<CategoryPath> categories) throws IOException {
-    int upto = ordinals.length; // since we add ordinals to IntsRef, iterate upto original length
+    int upto = ordinals.length; // since we may add ordinals to IntsRef, iterate upto original length
-    
+
-    if (ordinalPolicy == OrdinalPolicy.ALL_PARENTS) { // add all parents too
+    Iterator<CategoryPath> iter = categories.iterator();
-      for (int i = 0; i < upto; i++) {
+    for (int i = 0; i < upto; i++) {
-        int ordinal = ordinals.ints[i];
+      int ordinal = ordinals.ints[i];
+      CategoryPath cp = iter.next();
+      OrdinalPolicy op = clp.getOrdinalPolicy(cp.components[0]);
+      if (op != OrdinalPolicy.NO_PARENTS) {
+        // need to add parents too
         int parent = taxoWriter.getParent(ordinal);
         while (parent > 0) {
           ordinals.ints[ordinals.length++] = parent;
           parent = taxoWriter.getParent(parent);
         }
+        if (op == OrdinalPolicy.ALL_BUT_DIMENSION) { // discard the last added parent, which is the dimension
+          ordinals.length--;
+        }
       }
     }
     return ordinalsEncoder.encode(ordinals);

lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 
 import org.apache.lucene.facet.search.CategoryListIterator;
 import org.apache.lucene.facet.search.DocValuesCategoryListIterator;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.util.PartitionsUtils;
 import org.apache.lucene.util.encoding.DGapVInt8IntEncoder;
 import org.apache.lucene.util.encoding.IntDecoder;
@@ -35,25 +36,61 @@ import org.apache.lucene.util.encoding.UniqueValuesIntEncoder;
  */
 public class CategoryListParams {
 
-  /** OrdinalPolicy defines which ordinals are encoded for every document. */
+  /**
+   * Defines which category ordinals are encoded for every document. This also
+   * affects how category ordinals are aggregated, check the different policies
+   * for more details.
+   */
   public static enum OrdinalPolicy {
     /**
-     * Encodes only the ordinal of leaf nodes. That is, the category A/B/C will
+     * Encodes only the ordinals of leaf nodes. That is, for the category A/B/C,
-     * not encode the ordinals of A and A/B.
+     * the ordinals of A and A/B will not be encoded. This policy is efficient
+     * for hierarchical dimensions, as it reduces the number of ordinals that
+     * are visited per document. During faceted search, this policy behaves
+     * exactly like {@link #ALL_PARENTS}, and the counts of all path components
+     * will be computed as well.
      * 
      * <p>
      * <b>NOTE:</b> this {@link OrdinalPolicy} requires a special collector or
-     * accumulator, which will fix the parents' counts, unless you are not
+     * accumulator, which will fix the parents' counts.
-     * interested in the parents counts.
+     * 
+     * <p>
+     * <b>NOTE:</b> since only leaf nodes are encoded for the document, you
+     * should use this policy when the same document doesn't share two
+     * categories that have a mutual parent, or otherwise the counts will be
+     * wrong (the mutual parent will be over-counted). For example, if a
+     * document has the categories A/B/C and A/B/D, then with this policy the
+     * counts of "A" and "B" will be 2, which is wrong. If you intend to index
+     * hierarchical dimensions, with more than one category per document, you
+     * should use either {@link #ALL_PARENTS} or {@link #ALL_BUT_DIMENSION}.
      */
     NO_PARENTS,
     
     /**
      * Encodes the ordinals of all path components. That is, the category A/B/C
-     * will encode the ordinals of A and A/B as well. This is the default
+     * will encode the ordinals of A and A/B as well. If you don't require the
-     * {@link OrdinalPolicy}.
+     * dimension's count during search, consider using
+     * {@link #ALL_BUT_DIMENSION}.
      */
-    ALL_PARENTS
+    ALL_PARENTS,
+    
+    /**
+     * Encodes the ordinals of all path components except the dimension. The
+     * dimension of a category is defined to be the first components in
+     * {@link CategoryPath#components}. For the category A/B/C, the ordinal of
+     * A/B will be encoded as well, however not the ordinal of A.
+     * 
+     * <p>
+     * <b>NOTE:</b> when facets are aggregated, this policy behaves exactly like
+     * {@link #ALL_PARENTS}, except that the dimension is never counted. I.e. if
+     * you ask to count the facet "A", then while in {@link #ALL_PARENTS} you
+     * will get counts for "A" <u>and its children</u>, with this policy you
+     * will get counts for <u>only its children</u>. This policy is the default
+     * one, and makes sense for using with flat dimensions, whenever your
+     * application does not require the dimension's count. Otherwise, use
+     * {@link #ALL_PARENTS}.
+     */
+    ALL_BUT_DIMENSION
   }
   
   /** The default field used to store the facets information. */
@@ -63,7 +100,7 @@ public class CategoryListParams {
    * The default {@link OrdinalPolicy} that's used when encoding a document's
    * category ordinals.
    */
-  public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_PARENTS;
+  public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_BUT_DIMENSION;
   
   public final String field;
 
@@ -115,19 +152,15 @@ public class CategoryListParams {
       return false;
     }
     CategoryListParams other = (CategoryListParams) o;
-    if (this.hashCode != other.hashCode) {
+    if (hashCode != other.hashCode) {
       return false;
     }
-    
-    // The above hashcodes might equal each other in the case of a collision,
-    // so at this point only directly term equality testing will settle
-    // the equality test.
     return field.equals(other.field);
   }
 
   @Override
   public int hashCode() {
-    return this.hashCode;
+    return hashCode;
   }
 
   /** Create the {@link CategoryListIterator} for the specified partition. */
@@ -137,14 +170,18 @@ public class CategoryListParams {
     return new DocValuesCategoryListIterator(docValuesField, createEncoder().createMatchingDecoder());
   }
   
-  /** Returns the {@link OrdinalPolicy} to use for this {@link CategoryListParams}. */
+  /**
-  public OrdinalPolicy getOrdinalPolicy() {
+   * Returns the {@link OrdinalPolicy} to use for the given dimension. This
+   * {@link CategoryListParams} always returns {@link #DEFAULT_ORDINAL_POLICY}
+   * for all dimensions.
+   */
+  public OrdinalPolicy getOrdinalPolicy(String dimension) {
     return DEFAULT_ORDINAL_POLICY;
   }
   
   @Override
   public String toString() {
-    return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy();
+    return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy(null);
   }
   
-}
+}

lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionOrdinalPolicy.java

@@ -0,0 +1,55 @@
+package org.apache.lucene.facet.index.params;
+
+import java.util.Map;
+
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * A {@link CategoryListParams} which allow controlling the
+ * {@link CategoryListParams.OrdinalPolicy} used for each dimension. The
+ * dimension is specified as the first component in
+ * {@link CategoryPath#components}.
+ */
+public class PerDimensionOrdinalPolicy extends CategoryListParams {
+
+  private final Map<String,OrdinalPolicy> policies;
+  private final OrdinalPolicy defaultOP;
+  
+  public PerDimensionOrdinalPolicy(Map<String,OrdinalPolicy> policies) {
+    this(policies, DEFAULT_ORDINAL_POLICY);
+  }
+  
+  public PerDimensionOrdinalPolicy(Map<String,OrdinalPolicy> policies, OrdinalPolicy defaultOP) {
+    this.defaultOP = defaultOP;
+    this.policies = policies;
+  }
+
+  @Override
+  public OrdinalPolicy getOrdinalPolicy(String dimension) {
+    OrdinalPolicy op = policies.get(dimension);
+    return op == null ? defaultOP : op;
+  }
+  
+  @Override
+  public String toString() {
+    return super.toString() + " policies=" + policies;
+  }
+  
+}

lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java

@@ -81,7 +81,7 @@ import org.apache.lucene.util.encoding.DGapVInt8IntDecoder;
 public class CountingFacetsCollector extends FacetsCollector {
   
   private final FacetSearchParams fsp;
-  private final OrdinalPolicy ordinalPolicy;
+  private final CategoryListParams clp;
   private final TaxonomyReader taxoReader;
   private final BytesRef buf = new BytesRef(32);
   private final FacetArrays facetArrays;
@@ -101,8 +101,7 @@ public class CountingFacetsCollector extends FacetsCollector {
     assert assertParams(fsp) == null : assertParams(fsp);
     
     this.fsp = fsp;
-    CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
+    this.clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
-    this.ordinalPolicy = clp.getOrdinalPolicy();
     this.facetsField = clp.field;
     this.taxoReader = taxoReader;
     this.facetArrays = facetArrays;
@@ -209,21 +208,21 @@ public class CountingFacetsCollector extends FacetsCollector {
     }
   }
 
-  private void countParents(int[] parents) {
+  /**
-    // counts[0] is the count of ROOT, which we don't care about and counts[1]
+   * Computes the counts of ordinals under the given ordinal's tree, by
-    // can only update counts[0], so we don't bother to visit it too. also,
+   * recursively going down to leaf nodes and rollin up their counts (called
-    // since parents always have lower ordinals than their children, we traverse
+   * only with categories are indexing with OrdinalPolicy.NO_PARENTS).
-    // the array backwards. this also allows us to update just the immediate
+   */
-    // parent's count (actually, otherwise it would be a mistake).
+  private int rollupCounts(int ordinal, int[] children, int[] siblings) {
-    for (int i = counts.length - 1; i > 1; i--) {
+    int count = 0;
-      int count = counts[i];
+    while (ordinal != TaxonomyReader.INVALID_ORDINAL) {
-      if (count > 0) {
+      int childCount = counts[ordinal];
-        int parent = parents[i];
+      childCount += rollupCounts(children[ordinal], children, siblings);
-        if (parent != 0) {
+      counts[ordinal] = childCount;
-          counts[parent] += count;
+      count += childCount;
-        }
+      ordinal = siblings[ordinal];
-      }
     }
+    return count;
   }
 
   @Override
@@ -234,11 +233,6 @@ public class CountingFacetsCollector extends FacetsCollector {
       
       ParallelTaxonomyArrays arrays = taxoReader.getParallelTaxonomyArrays();
 
-      if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
-        // need to count parents
-        countParents(arrays.parents());
-      }
-
       // compute top-K
       final int[] children = arrays.children();
       final int[] siblings = arrays.siblings();
@@ -248,6 +242,12 @@ public class CountingFacetsCollector extends FacetsCollector {
         if (rootOrd == TaxonomyReader.INVALID_ORDINAL) { // category does not exist
           continue;
         }
+        OrdinalPolicy ordinalPolicy = clp.getOrdinalPolicy(fr.categoryPath.components[0]);
+        if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
+          // need to count parents
+          counts[rootOrd] += rollupCounts(children[rootOrd], children, siblings);
+        }
+        
         FacetResultNode root = new FacetResultNode();
         root.ordinal = rootOrd;
         root.label = fr.categoryPath;

lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java

@@ -43,13 +43,21 @@ public abstract class FacetsCollector extends Collector {
    * Returns the most optimized {@link FacetsCollector} for the given search
    * parameters. The returned {@link FacetsCollector} is guaranteed to satisfy
    * the requested parameters.
+   * 
+   * @throws IllegalArgumentException
+   *           if there is no built-in collector that can satisfy the search
+   *           parameters.
    */
   public static FacetsCollector create(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
     if (CountingFacetsCollector.assertParams(fsp) == null) {
       return new CountingFacetsCollector(fsp, taxoReader);
     }
     
-    return new StandardFacetsCollector(fsp, indexReader, taxoReader);
+    if (StandardFacetsCollector.assertParams(fsp) == null) {
+      return new StandardFacetsCollector(fsp, indexReader, taxoReader);
+    }
+    
+    throw new IllegalArgumentException("None of the built-in FacetsCollectors can handle the given search params");
   }
   
   /**

lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java

@@ -49,6 +49,17 @@ public class StandardFacetsCollector extends FacetsCollector {
   private List<FacetResult> results;
   private Object resultsGuard;
 
+  static String assertParams(FacetSearchParams fsp) {
+    // make sure none of the categories in the given FacetRequests was indexed with NO_PARENTS
+    for (FacetRequest fr : fsp.facetRequests) {
+      CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fr.categoryPath);
+      if (clp.getOrdinalPolicy(fr.categoryPath.components[0]) == OrdinalPolicy.NO_PARENTS) {
+        return "this collector does not support aggregating categories that were indexed with OrdinalPolicy.NO_PARENTS";
+      }
+    }
+    return null;
+  }
+  
   /**
    * Create a collector for accumulating facets while collecting documents
    * during search.
@@ -62,6 +73,7 @@ public class StandardFacetsCollector extends FacetsCollector {
    *          taxonomy containing the facets.
    */
   public StandardFacetsCollector(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
+    assert assertParams(facetSearchParams) == null : assertParams(facetSearchParams);
     facetsAccumulator = initFacetsAccumulator(facetSearchParams, indexReader, taxonomyReader);
     scoreDocIdCollector = initScoredDocCollector(facetSearchParams, indexReader, taxonomyReader);
     resultsGuard = new Object();

lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java

@@ -74,6 +74,7 @@ public class ScoredDocIdsUtils {
   
   /** Clear all deleted documents from a given open-bit-set according to a given reader */
   private static void clearDeleted(final IndexReader reader, final FixedBitSet set) throws IOException {
+    // TODO use BitsFilteredDocIdSet?
     
     // If there are no deleted docs
     if (!reader.hasDeletions()) {

lucene/facet/src/java/org/apache/lucene/facet/util/TaxonomyMergeUtils.java

@@ -1,9 +1,8 @@
-package org.apache.lucene.facet.example.merge;
+package org.apache.lucene.facet.util;
 
 import java.io.IOException;
 import java.util.List;
 
-import org.apache.lucene.facet.example.ExampleUtils;
 import org.apache.lucene.facet.index.OrdinalMappingAtomicReader;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
@@ -17,6 +16,7 @@ import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.MultiReader;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.Version;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -36,6 +36,7 @@ import org.apache.lucene.store.Directory;
  */
 
 /**
+ * Utility methods for merging index and taxonomy directories.
  * @lucene.experimental
  */
 public class TaxonomyMergeUtils {
@@ -52,8 +53,7 @@ public class TaxonomyMergeUtils {
    */
   public static void merge(Directory srcIndexDir, Directory srcTaxDir, Directory destIndexDir, Directory destTaxDir, 
       FacetIndexingParams params) throws IOException {
-    IndexWriter destIndexWriter = new IndexWriter(destIndexDir,
+    IndexWriter destIndexWriter = new IndexWriter(destIndexDir, new IndexWriterConfig(Version.LUCENE_42, null));
-        new IndexWriterConfig(ExampleUtils.EXAMPLE_VER, null));
     DirectoryTaxonomyWriter destTaxWriter = new DirectoryTaxonomyWriter(destTaxDir);
     merge(srcIndexDir, srcTaxDir, new MemoryOrdinalMap(), destIndexWriter, destTaxWriter, params);
     destTaxWriter.close();

lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java

@@ -6,6 +6,7 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
+import java.util.Iterator;
 import java.util.List;
 import java.util.Map;
 
@@ -17,6 +18,7 @@ import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.facet.index.FacetFields;
 import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.FacetRequest;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -44,6 +46,7 @@ import org.apache.lucene.util.Bits;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
 import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.collections.IntToObjectMap;
 import org.junit.AfterClass;
 import org.junit.BeforeClass;
 
@@ -73,7 +76,8 @@ public abstract class FacetTestBase extends FacetTestCase {
     SearchTaxoDirPair() {}
   }
   
-  private static HashMap<Integer, SearchTaxoDirPair> dirsPerPartitionSize;
+  private static IntToObjectMap<SearchTaxoDirPair> dirsPerPartitionSize;
+  private static IntToObjectMap<FacetIndexingParams> fipPerPartitionSize;
   private static File TEST_DIR;
   
   /** Documents text field. */
@@ -91,12 +95,15 @@ public abstract class FacetTestBase extends FacetTestCase {
   @BeforeClass
   public static void beforeClassFacetTestBase() {
     TEST_DIR = _TestUtil.getTempDir("facets");
-    dirsPerPartitionSize = new HashMap<Integer, FacetTestBase.SearchTaxoDirPair>();
+    dirsPerPartitionSize = new IntToObjectMap<FacetTestBase.SearchTaxoDirPair>();
+    fipPerPartitionSize = new IntToObjectMap<FacetIndexingParams>();
   }
   
   @AfterClass
   public static void afterClassFacetTestBase() throws Exception {
-    for (SearchTaxoDirPair pair : dirsPerPartitionSize.values()) {
+    Iterator<SearchTaxoDirPair> iter = dirsPerPartitionSize.iterator();
+    while (iter.hasNext()) {
+      SearchTaxoDirPair pair = iter.next();
       IOUtils.close(pair.searchDir, pair.taxoDir);
     }
   }
@@ -128,20 +135,16 @@ public abstract class FacetTestBase extends FacetTestCase {
     return DEFAULT_CONTENT[doc];
   }
   
-  /** Prepare index (in RAM) with single partition */
+  /** Prepare index (in RAM) with some documents and some facets. */
-  protected final void initIndex() throws Exception {
+  protected final void initIndex(FacetIndexingParams fip) throws Exception {
-    initIndex(Integer.MAX_VALUE);
+    initIndex(false, fip);
-  }
-  
-  /** Prepare index (in RAM) with some documents and some facets */
-  protected final void initIndex(int partitionSize) throws Exception {
-    initIndex(partitionSize, false);
   }
 
-  /** Prepare index (in RAM/Disk) with some documents and some facets */
+  /** Prepare index (in RAM/Disk) with some documents and some facets. */
-  protected final void initIndex(int partitionSize, boolean forceDisk) throws Exception {
+  protected final void initIndex(boolean forceDisk, FacetIndexingParams fip) throws Exception {
+    int partitionSize = fip.getPartitionSize();
     if (VERBOSE) {
-      System.out.println("Partition Size: " + partitionSize+"  forceDisk: "+forceDisk);
+      System.out.println("Partition Size: " + partitionSize + "  forceDisk: "+forceDisk);
     }
 
     SearchTaxoDirPair pair = dirsPerPartitionSize.get(Integer.valueOf(partitionSize));
@@ -158,7 +161,7 @@ public abstract class FacetTestBase extends FacetTestCase {
       RandomIndexWriter iw = new RandomIndexWriter(random(), pair.searchDir, getIndexWriterConfig(getAnalyzer()));
       TaxonomyWriter taxo = new DirectoryTaxonomyWriter(pair.taxoDir, OpenMode.CREATE);
       
-      populateIndex(iw, taxo, getFacetIndexingParams(partitionSize));
+      populateIndex(iw, taxo, fip);
       
       // commit changes (taxonomy prior to search index for consistency)
       taxo.commit();
@@ -182,14 +185,40 @@ public abstract class FacetTestBase extends FacetTestCase {
 
   /** Returns a {@link FacetIndexingParams} per the given partition size. */
   protected FacetIndexingParams getFacetIndexingParams(final int partSize) {
-    // several of our encoders don't support the value 0, 
+    return getFacetIndexingParams(partSize, false);
-    // which is one of the values encoded when dealing w/ partitions.
+  }
-    return new FacetIndexingParams() {
+  
-      @Override
+  /**
-      public int getPartitionSize() {
+   * Returns a {@link FacetIndexingParams} per the given partition size. If
-        return partSize;
+   * requested, then {@link OrdinalPolicy} will be set to
-      }
+   * {@link OrdinalPolicy#ALL_PARENTS}, otherwise it will randomize.
-    };
+   */
+  protected FacetIndexingParams getFacetIndexingParams(final int partSize, final boolean forceAllParents) {
+    FacetIndexingParams fip = fipPerPartitionSize.get(partSize);
+    if (fip == null) {
+      // randomize OrdinalPolicy. Since not all Collectors / Accumulators
+      // support NO_PARENTS, don't include it.
+      // TODO: once all code paths support NO_PARENTS, randomize it too.
+      CategoryListParams randomOP = new CategoryListParams() {
+        final OrdinalPolicy op = random().nextBoolean() ? OrdinalPolicy.ALL_BUT_DIMENSION : OrdinalPolicy.ALL_PARENTS;
+        @Override
+        public OrdinalPolicy getOrdinalPolicy(String dimension) {
+          return forceAllParents ? OrdinalPolicy.ALL_PARENTS : op;
+        }
+      };
+      
+      // several of our encoders don't support the value 0, 
+      // which is one of the values encoded when dealing w/ partitions,
+      // therefore don't randomize the encoder.
+      fip = new FacetIndexingParams(randomOP) {
+        @Override
+        public int getPartitionSize() {
+          return partSize;
+        }
+      };
+      fipPerPartitionSize.put(partSize, fip);
+    }
+    return fip;
   }
   
   /**

lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java

@@ -1,19 +1,7 @@
 package org.apache.lucene.facet;
 
-import java.io.IOException;
-
-import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.search.results.FacetResultNode;
-import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
-import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
-import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.IndexWriter;
-import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
 
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
@@ -34,68 +22,6 @@ import org.apache.lucene.util.LuceneTestCase;
 
 public class FacetTestUtils {
 
-  public static class IndexTaxonomyReaderPair {
-    public DirectoryReader indexReader;
-    public DirectoryTaxonomyReader taxReader;
-    public IndexSearcher indexSearcher;
-
-    public void close() throws IOException {
-      indexReader.close();
-      taxReader.close();
-    }
-
-  }
-
-  public static class IndexTaxonomyWriterPair {
-    public IndexWriter indexWriter;
-    public TaxonomyWriter taxWriter;
-
-    public void close() throws IOException {
-      indexWriter.close();
-      taxWriter.close();
-    }
-
-    public void commit() throws IOException {
-      indexWriter.commit();
-      taxWriter.commit();
-    }
-  }
-
-  public static Directory[][] createIndexTaxonomyDirs(int number) {
-    Directory[][] dirs = new Directory[number][2];
-    for (int i = 0; i < number; i++) {
-      dirs[i][0] = LuceneTestCase.newDirectory();
-      dirs[i][1] = LuceneTestCase.newDirectory();
-    }
-    return dirs;
-  }
-
-  public static IndexTaxonomyReaderPair[] createIndexTaxonomyReaderPair(Directory[][] dirs) throws IOException {
-    IndexTaxonomyReaderPair[] pairs = new IndexTaxonomyReaderPair[dirs.length];
-    for (int i = 0; i < dirs.length; i++) {
-      IndexTaxonomyReaderPair pair = new IndexTaxonomyReaderPair();
-      pair.indexReader = DirectoryReader.open(dirs[i][0]);
-      pair.indexSearcher = new IndexSearcher(pair.indexReader);
-      pair.taxReader = new DirectoryTaxonomyReader(dirs[i][1]);
-      pairs[i] = pair;
-    }
-    return pairs;
-  }
-  
-  public static IndexTaxonomyWriterPair[] createIndexTaxonomyWriterPair(Directory[][] dirs) throws IOException {
-    IndexTaxonomyWriterPair[] pairs = new IndexTaxonomyWriterPair[dirs.length];
-    for (int i = 0; i < dirs.length; i++) {
-      IndexTaxonomyWriterPair pair = new IndexTaxonomyWriterPair();
-      pair.indexWriter = new IndexWriter(dirs[i][0], new IndexWriterConfig(
-          LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(LuceneTestCase.random())));
-      pair.taxWriter = new DirectoryTaxonomyWriter(dirs[i][1]);
-      pair.indexWriter.commit();
-      pair.taxWriter.commit();
-      pairs[i] = pair;
-    }
-    return pairs;
-  }
-
   public static String toSimpleString(FacetResult fr) {
     StringBuilder sb = new StringBuilder();
     toSimpleString(0, sb, fr.getFacetResultNode(), "");

lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java

@@ -8,7 +8,6 @@ import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.example.merge.TaxonomyMergeUtils;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.FacetsCollector;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
@@ -18,6 +17,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
 import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.facet.util.TaxonomyMergeUtils;
 import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.RandomIndexWriter;

lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java

@@ -25,6 +25,8 @@ import org.apache.lucene.facet.FacetTestCase;
 import org.apache.lucene.facet.index.params.CategoryListParams;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
+import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
 import org.apache.lucene.facet.search.CategoryListIterator;
 import org.apache.lucene.facet.search.DrillDown;
 import org.apache.lucene.facet.search.FacetsCollector;
@@ -368,9 +370,23 @@ public class TestFacetsPayloadMigrationReader extends FacetTestCase {
     
     // set custom CLP fields for two dimensions and use the default ($facets) for the other two
     HashMap<CategoryPath,CategoryListParams> params = new HashMap<CategoryPath,CategoryListParams>();
-    params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]));
+    params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]) {
-    params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]));
+      @Override
-    FacetIndexingParams fip = new PerDimensionIndexingParams(params) {
+      public OrdinalPolicy getOrdinalPolicy(String dimension) {
+        return OrdinalPolicy.ALL_PARENTS;
+      }
+    });
+    params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]) {
+      @Override
+      public OrdinalPolicy getOrdinalPolicy(String dimension) {
+        return OrdinalPolicy.ALL_PARENTS;
+      }
+    });
+    
+    HashMap<String,OrdinalPolicy> policies = new HashMap<String,CategoryListParams.OrdinalPolicy>();
+    policies.put(DIMENSIONS[2], OrdinalPolicy.ALL_PARENTS);
+    policies.put(DIMENSIONS[3], OrdinalPolicy.ALL_PARENTS);
+    FacetIndexingParams fip = new PerDimensionIndexingParams(params, new PerDimensionOrdinalPolicy(policies)) {
       @Override
       public int getPartitionSize() {
         return partitionSize;

lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java

@@ -51,10 +51,9 @@ public abstract class BaseTestTopK extends FacetTestBase {
   private int nextInt;
 
   @Override
-  protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo,
+  protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo, FacetIndexingParams fip) throws IOException {
-      FacetIndexingParams iParams) throws IOException {
     currDoc = -1;
-    super.populateIndex(iw, taxo, iParams);
+    super.populateIndex(iw, taxo, fip);
   }
   
   /** prepare the next random int */
@@ -94,17 +93,13 @@ public abstract class BaseTestTopK extends FacetTestBase {
     return Arrays.asList(cp);
   }
 
-  protected FacetSearchParams searchParamsWithRequests(int numResults) {
+  protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
-    return searchParamsWithRequests(numResults, Integer.MAX_VALUE);
-  }
-  
-  protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
     List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
     facetRequests.add(new CountFacetRequest(new CategoryPath("a"), numResults));
     facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
     facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
     facetRequests.add(new CountFacetRequest(new CategoryPath("a", "2",  "26", "267"), numResults));
-    return getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
+    return getFacetSearchParams(facetRequests, fip);
   }
 
   @Override

lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java

@@ -16,8 +16,9 @@ import org.apache.lucene.document.StringField;
 import org.apache.lucene.facet.FacetTestCase;
 import org.apache.lucene.facet.index.FacetFields;
 import org.apache.lucene.facet.index.params.CategoryListParams;
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
-import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
+import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
 import org.apache.lucene.facet.search.params.FacetRequest;
 import org.apache.lucene.facet.search.params.FacetRequest.SortBy;
@@ -146,13 +147,11 @@ public class CountingFacetsCollectorTest extends FacetTestCase {
         termExpectedCounts.put(cp, termExpectedCounts.get(cp) + 1);
       }
     }
-    // add 1 to each dimension
+    // add 1 to each NO_PARENTS dimension
-    allExpectedCounts.put(CP_A, allExpectedCounts.get(CP_A) + 1);
     allExpectedCounts.put(CP_B, allExpectedCounts.get(CP_B) + 1);
     allExpectedCounts.put(CP_C, allExpectedCounts.get(CP_C) + 1);
     allExpectedCounts.put(CP_D, allExpectedCounts.get(CP_D) + 1);
     if (updateTermExpectedCounts) {
-      termExpectedCounts.put(CP_A, termExpectedCounts.get(CP_A) + 1);
       termExpectedCounts.put(CP_B, termExpectedCounts.get(CP_B) + 1);
       termExpectedCounts.put(CP_C, termExpectedCounts.get(CP_C) + 1);
       termExpectedCounts.put(CP_D, termExpectedCounts.get(CP_D) + 1);
@@ -252,19 +251,13 @@ public class CountingFacetsCollectorTest extends FacetTestCase {
     conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges, so we can control the index segments
     IndexWriter indexWriter = new IndexWriter(indexDir, conf);
     TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
-    CategoryListParams allParents = new CategoryListParams();
+
-    CategoryListParams noParents = new CategoryListParams("no_parents") {
+    Map<String,OrdinalPolicy> policies = new HashMap<String,CategoryListParams.OrdinalPolicy>();
-      @Override
+    policies.put(CP_B.components[0], OrdinalPolicy.ALL_PARENTS);
-      public OrdinalPolicy getOrdinalPolicy() {
+    policies.put(CP_C.components[0], OrdinalPolicy.NO_PARENTS);
-        return OrdinalPolicy.NO_PARENTS;
+    policies.put(CP_D.components[0], OrdinalPolicy.NO_PARENTS);
-      }
+    CategoryListParams clp = new PerDimensionOrdinalPolicy(policies);
-    };
+    fip = new FacetIndexingParams(clp);
-    Map<CategoryPath,CategoryListParams> params = new HashMap<CategoryPath,CategoryListParams>();
-    params.put(CP_A, allParents);
-    params.put(CP_B, allParents);
-    params.put(CP_C, noParents);
-    params.put(CP_D, noParents);
-    fip = new PerDimensionIndexingParams(params);
     
     allExpectedCounts = newCounts();
     termExpectedCounts = newCounts();

lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java

@@ -104,9 +104,9 @@ public class TestDemoFacets extends FacetTestCase {
     // Retrieve & verify results:
     List<FacetResult> results = c.getFacetResults();
     assertEquals(2, results.size());
-    assertEquals("Publish Date (5)\n  2012 (2)\n  2010 (2)\n  1999 (1)\n",
+    assertEquals("Publish Date (0)\n  2012 (2)\n  2010 (2)\n  1999 (1)\n",
         FacetTestUtils.toSimpleString(results.get(0)));
-    assertEquals("Author (5)\n  Lisa (2)\n  Frank (1)\n  Susan (1)\n  Bob (1)\n",
+    assertEquals("Author (0)\n  Lisa (2)\n  Frank (1)\n  Susan (1)\n  Bob (1)\n",
         FacetTestUtils.toSimpleString(results.get(1)));
 
     
@@ -117,7 +117,7 @@ public class TestDemoFacets extends FacetTestCase {
     searcher.search(q2, c);
     results = c.getFacetResults();
     assertEquals(1, results.size());
-    assertEquals("Author (2)\n  Lisa (1)\n  Bob (1)\n",
+    assertEquals("Author (0)\n  Lisa (1)\n  Bob (1)\n",
         FacetTestUtils.toSimpleString(results.get(0)));
 
     // Smoke test PrintTaxonomyStats:

lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java

@@ -14,6 +14,7 @@ import org.junit.Before;
 import org.junit.Test;
 
 import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.FacetsAccumulator;
 import org.apache.lucene.facet.search.ScoredDocIDs;
 import org.apache.lucene.facet.search.ScoredDocIdCollector;
@@ -48,11 +49,14 @@ import org.apache.lucene.facet.taxonomy.CategoryPath;
  */
 public class TestFacetsAccumulatorWithComplement extends FacetTestBase {
   
+  private FacetIndexingParams fip;
+  
   @Override
   @Before
   public void setUp() throws Exception {
     super.setUp();
-    initIndex();
+    fip = getFacetIndexingParams(Integer.MAX_VALUE);
+    initIndex(fip);
   }
   
   @Override
@@ -125,7 +129,7 @@ public class TestFacetsAccumulatorWithComplement extends FacetTestBase {
   
   /** compute facets with certain facet requests and docs */
   private List<FacetResult> findFacets(ScoredDocIDs sDocids, boolean withComplement) throws IOException {
-    FacetSearchParams fsp = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(new CategoryPath("root","a"), 10));
+    FacetSearchParams fsp = new FacetSearchParams(fip, new CountFacetRequest(new CategoryPath("root","a"), 10));
     FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(fsp, indexReader, taxoReader);
     
     fAccumulator.setComplementThreshold(

lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java

@@ -14,7 +14,6 @@ import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.TextField;
 import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.FacetTestUtils;
 import org.apache.lucene.facet.index.FacetFields;
 import org.apache.lucene.facet.index.params.CategoryListParams;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
@@ -77,12 +76,14 @@ public class TestMultipleCategoryLists extends FacetTestCase {
   
   @Test
   public void testDefault() throws Exception {
-    Directory[][] dirs = getDirs();
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    
     // create and open an index writer
-    RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+    RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
     // create and open a taxonomy writer
-    TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
 
     PerDimensionIndexingParams iParams = new PerDimensionIndexingParams(Collections.<CategoryPath, CategoryListParams>emptyMap());
 
@@ -92,7 +93,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     tw.commit();
 
     // prepare index reader and taxonomy.
-    TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+    TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
 
     // prepare searcher to search against
     IndexSearcher searcher = newSearcher(ir);
@@ -105,17 +106,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     assertOrdinalsExist("$facets", ir);
 
     IOUtils.close(tr, ir, iw, tw);
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
   }
 
   @Test
   public void testCustom() throws Exception {
-    Directory[][] dirs = getDirs();
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    
     // create and open an index writer
-    RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+    RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
     // create and open a taxonomy writer
-    TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
 
     PerDimensionIndexingParams iParams = new PerDimensionIndexingParams(
         Collections.singletonMap(new CategoryPath("Author"), new CategoryListParams("$author")));
@@ -125,7 +128,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     tw.commit();
 
     // prepare index reader and taxonomy.
-    TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+    TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
 
     // prepare searcher to search against
     IndexSearcher searcher = newSearcher(ir);
@@ -139,17 +142,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     assertOrdinalsExist("$author", ir);
 
     IOUtils.close(tr, ir, iw, tw);
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
   }
 
   @Test
   public void testTwoCustomsSameField() throws Exception {
-    Directory[][] dirs = getDirs();
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    
     // create and open an index writer
-    RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+    RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
     // create and open a taxonomy writer
-    TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
 
     Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
     paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$music"));
@@ -161,7 +166,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     tw.commit();
 
     // prepare index reader and taxonomy.
-    TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+    TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
 
     // prepare searcher to search against
     IndexSearcher searcher = newSearcher(ir);
@@ -176,7 +181,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     assertOrdinalsExist("$music", ir);
 
     IOUtils.close(tr, ir, iw, tw);
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
   }
 
   private void assertOrdinalsExist(String field, IndexReader ir) throws IOException {
@@ -191,12 +196,14 @@ public class TestMultipleCategoryLists extends FacetTestCase {
 
   @Test
   public void testDifferentFieldsAndText() throws Exception {
-    Directory[][] dirs = getDirs();
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+
     // create and open an index writer
-    RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+    RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
     // create and open a taxonomy writer
-    TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
 
     Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
     paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$bands"));
@@ -208,7 +215,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     tw.commit();
 
     // prepare index reader and taxonomy.
-    TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+    TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
 
     // prepare searcher to search against
     IndexSearcher searcher = newSearcher(ir);
@@ -222,17 +229,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     assertOrdinalsExist("$composers", ir);
 
     IOUtils.close(tr, ir, iw, tw);
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
   }
 
   @Test
   public void testSomeSameSomeDifferent() throws Exception {
-    Directory[][] dirs = getDirs();
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
+    
     // create and open an index writer
-    RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
+    RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
         TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
     // create and open a taxonomy writer
-    TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
+    TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
 
     Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
     paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$music"));
@@ -246,7 +255,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     tw.commit();
 
     // prepare index reader and taxonomy.
-    TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
+    TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
 
     // prepare searcher to search against
     IndexSearcher searcher = newSearcher(ir);
@@ -259,11 +268,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     assertOrdinalsExist("$literature", ir);
 
     IOUtils.close(tr, ir, iw, tw);
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
-  }
-
-  private Directory[][] getDirs() {
-    return FacetTestUtils.createIndexTaxonomyDirs(1);
   }
 
   private void assertCorrectResults(FacetsCollector facetsCollector) throws IOException {
@@ -274,7 +279,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     Iterable<? extends FacetResultNode> subResults = resNode.subResults;
     Iterator<? extends FacetResultNode> subIter = subResults.iterator();
 
-    checkResult(resNode, "Band", 5.0);
     checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
     checkResult(subIter.next(), "Band/Punk", 1.0);
 
@@ -283,7 +287,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     subResults = resNode.subResults;
     subIter = subResults.iterator();
 
-    checkResult(resNode, "Band", 5.0);
     checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
     checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
     checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
@@ -297,7 +300,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     subResults = resNode.subResults;
     subIter = subResults.iterator();
 
-    checkResult(resNode, "Author", 3.0);
     checkResult(subIter.next(), "Author/Kurt Vonnegut", 1.0);
     checkResult(subIter.next(), "Author/Stephen King", 1.0);
     checkResult(subIter.next(), "Author/Mark Twain", 1.0);
@@ -307,7 +309,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
     subResults = resNode.subResults;
     subIter = subResults.iterator();
 
-    checkResult(resNode, "Band/Rock & Pop", 4.0);
     checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
     checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
     checkResult(subIter.next(), "Band/Rock & Pop/U2", 1.0);

lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java

@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search;
 import java.util.List;
 
 import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.FacetsCollector;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
@@ -31,18 +32,21 @@ import org.junit.Before;
 
 public class TestSameRequestAccumulation extends FacetTestBase {
   
+  private FacetIndexingParams fip;
+  
   @Override
   @Before
   public void setUp() throws Exception {
     super.setUp();
-    initIndex();
+    fip = getFacetIndexingParams(Integer.MAX_VALUE);
+    initIndex(fip);
   }
   
   // Following LUCENE-4461 - ensure requesting the (exact) same request more
   // than once does not alter the results
   public void testTwoSameRequests() throws Exception {
     final CountFacetRequest facetRequest = new CountFacetRequest(new CategoryPath("root"), 10);
-    FacetSearchParams fsp = new FacetSearchParams(facetRequest);
+    FacetSearchParams fsp = new FacetSearchParams(fip, facetRequest);
     
     FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
     searcher.search(new MatchAllDocsQuery(), fc);
@@ -50,7 +54,7 @@ public class TestSameRequestAccumulation extends FacetTestBase {
     final String expected = fc.getFacetResults().get(0).toString();
 
     // now add the same facet request with duplicates (same instance and same one)
-    fsp = new FacetSearchParams(facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
+    fsp = new FacetSearchParams(fip, facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
 
     // make sure the search params holds 3 requests now
     assertEquals(3, fsp.facetRequests.size());

lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java

@@ -5,6 +5,7 @@ import java.util.Arrays;
 import java.util.List;
 
 import org.apache.lucene.facet.FacetTestBase;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
 import org.apache.lucene.facet.search.params.ScoreFacetRequest;
@@ -37,11 +38,14 @@ import org.junit.Test;
 /** Test ScoredDocIdCollector. */
 public class TestScoredDocIdCollector extends FacetTestBase {
 
+  private FacetIndexingParams fip;
+  
   @Override
   @Before
   public void setUp() throws Exception {
     super.setUp();
-    initIndex();
+    fip = getFacetIndexingParams(Integer.MAX_VALUE);
+    initIndex(fip);
   }
 
   @Override
@@ -73,8 +77,8 @@ public class TestScoredDocIdCollector extends FacetTestBase {
 
     // verify by facet values
     CategoryPath cp = new CategoryPath("root","a");
-    FacetSearchParams countFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(cp, 10));
+    FacetSearchParams countFSP = new FacetSearchParams(fip, new CountFacetRequest(cp, 10));
-    FacetSearchParams scoreFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new ScoreFacetRequest(cp, 10));
+    FacetSearchParams scoreFSP = new FacetSearchParams(fip, new ScoreFacetRequest(cp, 10));
     
     List<FacetResult> countRes = findFacets(scoredDocIDs, countFSP);
     List<FacetResult> scoreRes = findFacets(scoredDocIDs, scoreFSP);
@@ -101,10 +105,8 @@ public class TestScoredDocIdCollector extends FacetTestBase {
   }
 
   // compute facets with certain facet requests and docs
-  private List<FacetResult> findFacets(ScoredDocIDs sDocids,
+  private List<FacetResult> findFacets(ScoredDocIDs sDocids, FacetSearchParams facetSearchParams) throws IOException {
-      FacetSearchParams facetSearchParams) throws IOException {
+    FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(facetSearchParams, indexReader, taxoReader);
-    FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(
-        facetSearchParams, indexReader, taxoReader);
     List<FacetResult> res = fAccumulator.accumulate(sDocids);
 
     // Results are ready, printing them...

lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java

@@ -113,7 +113,6 @@ public class TestStandardFacetsAccumulator extends FacetTestCase {
     List<FacetResult> results = fc.getFacetResults();
     assertEquals("received too many facet results", 1, results.size());
     FacetResultNode frn = results.get(0).getFacetResultNode();
-    assertEquals("wrong weight for \"A\"", 4, (int) frn.value);
     assertEquals("wrong number of children", 2, frn.subResults.size());
     for (FacetResultNode node : frn.subResults) {
       assertEquals("wrong weight for child " + node.label, 2, (int) node.value);

lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java

@@ -181,7 +181,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
       boolean hasDoctor = "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
       assertEquals(9, fr.getNumValidDescendants());
       FacetResultNode parentRes = fr.getFacetResultNode();
-      assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
       assertEquals(2, parentRes.subResults.size());
       // two nodes sorted by descending values: a/b with 8  and a/c with 6
       // a/b has two children a/b/2 with value 3, and a/b/1 with value 2. 
@@ -217,7 +216,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
       hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
       assertEquals(9, fr.getNumValidDescendants());
       parentRes = fr.getFacetResultNode();
-      assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
       assertEquals(2, parentRes.subResults.size());
       // two nodes sorted by descending values: a/b with 8  and a/c with 6
       // a/b has two children a/b/2 with value 3, and a/b/1 with value 2. 
@@ -234,7 +232,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
       hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
       assertEquals(4, fr.getNumValidDescendants(), 4);
       parentRes = fr.getFacetResultNode();
-      assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
       assertEquals(2, parentRes.subResults.size());
       // two nodes sorted by descending values: 
       // a/b with value 8 and a/c with value 6

lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java

@@ -4,6 +4,8 @@ import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
 
+import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.CountFacetRequest;
 import org.apache.lucene.facet.search.params.FacetRequest;
 import org.apache.lucene.facet.search.params.FacetRequest.ResultMode;
@@ -73,7 +75,9 @@ public class TestTopKResultsHandler extends BaseTestTopK {
   @Test
   public void testSimple() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      OrdinalPolicy op = fip.getCategoryListParams(null).getOrdinalPolicy(null);
+      initIndex(fip);
 
       List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
       facetRequests.add(new CountFacetRequest(new CategoryPath("a"), 100));
@@ -87,8 +91,8 @@ public class TestTopKResultsHandler extends BaseTestTopK {
       facetRequests.add(new CountFacetRequest(new CategoryPath("a", "c"), 100));
       
       // do different facet counts and compare to control
-      FacetSearchParams sParams = getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
+      FacetSearchParams sParams = getFacetSearchParams(facetRequests, fip);
-
+      
       FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
         @Override
         protected FacetsAccumulator initFacetsAccumulator(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
@@ -100,17 +104,21 @@ public class TestTopKResultsHandler extends BaseTestTopK {
       
       searcher.search(new MatchAllDocsQuery(), fc);
       List<FacetResult> facetResults = fc.getFacetResults();
-
+      
       FacetResult fr = facetResults.get(0);
       FacetResultNode parentRes = fr.getFacetResultNode();
-      assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      }
       FacetResultNode[] frn = resultNodesAsArray(parentRes);
       assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
       assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
 
       fr = facetResults.get(1);
       parentRes = fr.getFacetResultNode();
-      assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
       assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
@@ -121,7 +129,9 @@ public class TestTopKResultsHandler extends BaseTestTopK {
 
       fr = facetResults.get(2);
       parentRes = fr.getFacetResultNode();
-      assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(2.0, frn[0].value, Double.MIN_VALUE);
       assertEquals(2.0, frn[1].value, Double.MIN_VALUE);
@@ -130,13 +140,17 @@ public class TestTopKResultsHandler extends BaseTestTopK {
 
       fr = facetResults.get(3);
       parentRes = fr.getFacetResultNode();
-      assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(0, frn.length);
 
       fr = facetResults.get(4);
       parentRes = fr.getFacetResultNode();
-      assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+      if (op == OrdinalPolicy.ALL_PARENTS) {
+        assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
+      }
       frn = resultNodesAsArray(parentRes);
       assertEquals(1.0, frn[0].value, Double.MIN_VALUE);
       closeAll();
@@ -149,12 +163,12 @@ public class TestTopKResultsHandler extends BaseTestTopK {
   @Test
   public void testGetMaxIntFacets() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      initIndex(fip);
 
       // do different facet counts and compare to control
       CategoryPath path = new CategoryPath("a", "b");
-      FacetSearchParams sParams = getFacetSearchParams(getFacetIndexingParams(partitionSize), 
+      FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, Integer.MAX_VALUE));
-          new CountFacetRequest(path, Integer.MAX_VALUE));
 
       FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
         @Override
@@ -174,7 +188,7 @@ public class TestTopKResultsHandler extends BaseTestTopK {
 
       // As a control base results, ask for top-1000 results
       FacetSearchParams sParams2 = getFacetSearchParams(
-          getFacetIndexingParams(partitionSize), new CountFacetRequest(path, Integer.MAX_VALUE));
+          fip, new CountFacetRequest(path, Integer.MAX_VALUE));
 
       FacetsCollector fc2 = new StandardFacetsCollector(sParams2, indexReader, taxoReader) {
         @Override
@@ -207,12 +221,11 @@ public class TestTopKResultsHandler extends BaseTestTopK {
   @Test
   public void testSimpleSearchForNonexistentFacet() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      initIndex(fip);
 
       CategoryPath path = new CategoryPath("Miau Hattulla");
-      FacetSearchParams sParams = getFacetSearchParams(
+      FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, 10));
-          getFacetIndexingParams(partitionSize),
-          new CountFacetRequest(path, 10));
 
       FacetsCollector fc = FacetsCollector.create(sParams, indexReader, taxoReader);
       

lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java

@@ -4,6 +4,7 @@ import java.io.IOException;
 import java.util.HashMap;
 import java.util.List;
 
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.params.FacetSearchParams;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.search.results.FacetResultNode;
@@ -32,10 +33,10 @@ import org.junit.Test;
 
 public class TestTopKResultsHandlerRandom extends BaseTestTopK {
   
-  private List<FacetResult> countFacets(int partitionSize, int numResults, final boolean doComplement)
+  private List<FacetResult> countFacets(FacetIndexingParams fip, int numResults, final boolean doComplement)
       throws IOException {
     Query q = new MatchAllDocsQuery();
-    FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, partitionSize);
+    FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, fip);
     FacetsCollector fc = new StandardFacetsCollector(facetSearchParams, indexReader, taxoReader) {
       @Override
       protected FacetsAccumulator initFacetsAccumulator(
@@ -59,7 +60,8 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
   @Test
   public void testTopCountsOrder() throws Exception {
     for (int partitionSize : partitionSizes) {
-      initIndex(partitionSize);
+      FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
+      initIndex(fip);
       
       /*
        * Try out faceted search in it's most basic form (no sampling nor complement
@@ -67,7 +69,7 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
        * being indexed, and later on an "over-all" faceted search is performed. The
        * results are checked against the DF of each facet by itself
        */
-      List<FacetResult> facetResults = countFacets(partitionSize, 100000, false);
+      List<FacetResult> facetResults = countFacets(fip, 100000, false);
       assertCountsAndCardinality(facetCountsTruth(), facetResults);
       
       /*
@@ -77,10 +79,10 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
        * place in here. The results are checked against the a regular (a.k.a
        * no-complement, no-sampling) faceted search with the same parameters.
        */
-      facetResults = countFacets(partitionSize, 100000, true);
+      facetResults = countFacets(fip, 100000, true);
       assertCountsAndCardinality(facetCountsTruth(), facetResults);
       
-      List<FacetResult> allFacetResults = countFacets(partitionSize, 100000, false);
+      List<FacetResult> allFacetResults = countFacets(fip, 100000, false);
       
       HashMap<String,Integer> all = new HashMap<String,Integer>();
       int maxNumNodes = 0;
@@ -108,7 +110,7 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
         if (VERBOSE) {
           System.out.println("-------  verify for "+n+" top results");
         }
-        List<FacetResult> someResults = countFacets(partitionSize, n, false);
+        List<FacetResult> someResults = countFacets(fip, n, false);
         k = 0;
         for (FacetResult fr : someResults) {
           FacetResultNode topResNode = fr.getFacetResultNode();

lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java

@@ -3,12 +3,20 @@ package org.apache.lucene.facet.search;
 import java.io.File;
 import java.io.IOException;
 import java.util.Arrays;
+import java.util.Collections;
 
+import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.FacetTestUtils;
+import org.apache.lucene.facet.index.FacetFields;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
+import org.apache.lucene.facet.index.params.CategoryListParams;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
+import org.apache.lucene.facet.taxonomy.CategoryPath;
+import org.apache.lucene.facet.taxonomy.TaxonomyReader;
+import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
+import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util._TestUtil;
@@ -33,9 +41,9 @@ import org.junit.Test;
 
 public class TestTotalFacetCounts extends FacetTestCase {
 
-  private static void initCache(int numEntries) {
+  private static void initCache() {
     TotalFacetCountsCache.getSingleton().clear();
-    TotalFacetCountsCache.getSingleton().setCacheSize(numEntries); // Set to keep one in mem
+    TotalFacetCountsCache.getSingleton().setCacheSize(1); // Set to keep one in mem
   }
 
   @Test
@@ -48,51 +56,60 @@ public class TestTotalFacetCounts extends FacetTestCase {
   }
 
   private void doTestWriteRead(final int partitionSize) throws IOException {
-    initCache(1);
+    initCache();
 
-    // Create temporary RAMDirectories
+    Directory indexDir = newDirectory();
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+    Directory taxoDir = newDirectory();
-    // Create our index/taxonomy writers
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+    
     FacetIndexingParams iParams = new FacetIndexingParams() {
       @Override
       public int getPartitionSize() {
         return partitionSize;
       }
+      
+      @Override
+      public CategoryListParams getCategoryListParams(CategoryPath category) {
+        return new CategoryListParams() {
+          @Override
+          public OrdinalPolicy getOrdinalPolicy(String dimension) {
+            return OrdinalPolicy.ALL_PARENTS;
+          }
+        };
+      }
     };
     // The counts that the TotalFacetCountsArray should have after adding
     // the below facets to the index.
     int[] expectedCounts = new int[] { 0, 3, 1, 3, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1 };
-    
+    String[] categories = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };
-    // Add a facet to the index
+
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+    FacetFields facetFields = new FacetFields(taxoWriter, iParams);
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
+    for (String cat : categories) {
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "e");
+      Document doc = new Document();
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "d");
+      facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "g");
+      indexWriter.addDocument(doc);
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "z");
+    }
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "a");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "1", "2");
-    TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "c");
 
     // Commit Changes
-    writers[0].close();
+    IOUtils.close(indexWriter, taxoWriter);
 
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader = DirectoryReader.open(indexDir);
+    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
     
     int[] intArray = new int[iParams.getPartitionSize()];
 
     TotalFacetCountsCache tfcc = TotalFacetCountsCache.getSingleton();
     File tmpFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
-    tfcc.store(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    tfcc.store(tmpFile, indexReader, taxoReader, iParams);
     tfcc.clear(); // not really required because TFCC overrides on load(), but in the test we need not rely on this.
-    tfcc.load(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    tfcc.load(tmpFile, indexReader, taxoReader, iParams);
     
     // now retrieve the one just loaded
-    TotalFacetCounts totalCounts = tfcc.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TotalFacetCounts totalCounts = tfcc.getTotalCounts(indexReader, taxoReader, iParams);
 
     int partition = 0;
-    for (int i=0; i<expectedCounts.length; i+=partitionSize) {
+    for (int i = 0; i < expectedCounts.length; i += partitionSize) {
       totalCounts.fillTotalCountsForPartition(intArray, partition);
       int[] partitionExpectedCounts = new int[partitionSize];
       int nToCopy = Math.min(partitionSize,expectedCounts.length-i);
@@ -103,8 +120,8 @@ public class TestTotalFacetCounts extends FacetTestCase {
           Arrays.equals(partitionExpectedCounts, intArray));
       ++partition;
     }
-    readers[0].close();
+    IOUtils.close(indexReader, taxoReader);
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
     tmpFile.delete();
   }
 

lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java

@@ -4,21 +4,17 @@ import java.io.File;
 import java.io.IOException;
 import java.util.Collections;
 import java.util.List;
+import java.util.Random;
 
 import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.analysis.MockTokenizer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetTestCase;
-import org.apache.lucene.facet.FacetTestUtils;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
-import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
-import org.apache.lucene.facet.example.ExampleResult;
-import org.apache.lucene.facet.example.TestMultiCLExample;
-import org.apache.lucene.facet.example.multiCL.MultiCLIndexer;
-import org.apache.lucene.facet.example.multiCL.MultiCLSearcher;
 import org.apache.lucene.facet.index.FacetFields;
 import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.TotalFacetCounts.CreationType;
+import org.apache.lucene.facet.search.params.CountFacetRequest;
+import org.apache.lucene.facet.search.params.FacetSearchParams;
 import org.apache.lucene.facet.search.results.FacetResult;
 import org.apache.lucene.facet.search.results.FacetResultNode;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
@@ -30,6 +26,8 @@ import org.apache.lucene.index.DirectoryReader;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchAllDocsQuery;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.MockDirectoryWrapper;
 import org.apache.lucene.util.IOUtils;
@@ -86,8 +84,8 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
   }
 
   /** Utility method to add a document and facets to an index/taxonomy. */
-  static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
+  private static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
-                        TaxonomyWriter tw, String... strings) throws IOException {
+      TaxonomyWriter tw, String... strings) throws IOException {
     Document doc = new Document();
     FacetFields facetFields = new FacetFields(tw, iParams);
     facetFields.addFields(doc, Collections.singletonList(new CategoryPath(strings)));
@@ -95,7 +93,7 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
   }
 
   /** Clears the cache and sets its size to one. */
-  static void initCache() {
+  private static void initCache() {
     TFC.clear();
     TFC.setCacheSize(1); // Set to keep one in memory
   }
@@ -107,37 +105,35 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
     initCache();
   }
 
-  /** runs a few instances of {@link MultiCLSearcher} in parallel */
+  /** runs few searches in parallel */
   public void testGeneralSynchronization() throws Exception {
-    int numIters = atLeast(2);
+    int numIters = atLeast(4);
+    Random random = random();
     for (int i = 0; i < numIters; i++) {
-      doTestGeneralSynchronization(_TestUtil.nextInt(random(), 2, 4),
+      int numThreads = random.nextInt(3) + 2; // 2-4
-                                  random().nextBoolean() ? -1 : _TestUtil.nextInt(random(), 1, 10),
+      int sleepMillis = random.nextBoolean() ? -1 : random.nextInt(10) + 1 /*1-10*/;
-                                  _TestUtil.nextInt(random(), 0, 3));
+      int cacheSize = random.nextInt(4); // 0-3
+      doTestGeneralSynchronization(numThreads, sleepMillis, cacheSize);
     }
   }
 
-  /**
+  private static final String[] CATEGORIES = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };
-   * Run many instances of {@link MultiCLSearcher} in parallel, results should
-   * be sane. Each instance has a random delay for reading bytes, to ensure
-   * that threads finish in different order than started.
-   */
-  @Test @Nightly
-  public void testGeneralSynchronizationBig() throws Exception {
-    int[] numThreads = new int[] { 2, 3, 5, 8 };
-    int[] sleepMillis = new int[] { -1, 1, 20, 33 };
-    int[] cacheSize = new int[] { 0,1,2,3,5 };
-    for (int size : cacheSize) {
-      for (int sleep : sleepMillis) {
-        for (int nThreads : numThreads) {
-          doTestGeneralSynchronization(nThreads, sleep, size);
-        }
-      }
-    }
-  }
 
-  private void doTestGeneralSynchronization(int numThreads, int sleepMillis,
+  private void index(Directory indexDir, Directory taxoDir) throws IOException {
-      int cacheSize) throws Exception {
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
+    FacetFields facetFields = new FacetFields(taxoWriter);
+    
+    for (String cat : CATEGORIES) {
+      Document doc = new Document();
+      facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
+      indexWriter.addDocument(doc);
+    }
+    
+    IOUtils.close(indexWriter, taxoWriter);
+  }
+  
+  private void doTestGeneralSynchronization(int numThreads, int sleepMillis, int cacheSize) throws Exception {
     TFC.setCacheSize(cacheSize);
     SlowRAMDirectory slowIndexDir = new SlowRAMDirectory(-1, random());
     MockDirectoryWrapper indexDir = new MockDirectoryWrapper(random(), slowIndexDir);
@@ -145,7 +141,7 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
     MockDirectoryWrapper taxoDir = new MockDirectoryWrapper(random(), slowTaxoDir);
 
     // Index documents without the "slowness"
-    MultiCLIndexer.index(indexDir, taxoDir);
+    index(indexDir, taxoDir);
 
     slowIndexDir.setSleepMillis(sleepMillis);
     slowTaxoDir.setSleepMillis(sleepMillis);
@@ -161,80 +157,64 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
       private IndexReader indexReader;
       private TaxonomyReader taxoReader;
 
-      public Multi(IndexReader indexReader, TaxonomyReader taxoReader,
+      public Multi(IndexReader indexReader, TaxonomyReader taxoReader, FacetIndexingParams iParams) {
-                    FacetIndexingParams iParams) {
         this.indexReader = indexReader;
         this.taxoReader = taxoReader;
         this.iParams = iParams;
       }
 
-      public ExampleResult getResults() {
+      public List<FacetResult> getResults() {
-        ExampleResult exampleRes = new ExampleResult();
+        return results;
-        exampleRes.setFacetResults(results);
-        return exampleRes;
       }
 
       @Override
       public void run() {
         try {
-          results = MultiCLSearcher.searchWithFacets(indexReader, taxoReader, iParams);
+          FacetSearchParams fsp = new FacetSearchParams(iParams, new CountFacetRequest(new CategoryPath("a"), 10),
+              new CountFacetRequest(new CategoryPath("b"), 10));
+          IndexSearcher searcher = new IndexSearcher(indexReader);
+          FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
+          searcher.search(new MatchAllDocsQuery(), fc);
+          results = fc.getFacetResults();
         } catch (Exception e) {
           throw new RuntimeException(e);
         }
       }
     }
 
-    // Instantiate threads, but do not start them
     Multi[] multis = new Multi[numThreads];
-    for (int i = 0; i < numThreads - 1; i++) {
+    for (int i = 0; i < numThreads; i++) {
-      multis[i] = new Multi(slowIndexReader, slowTaxoReader, MultiCLIndexer.MULTI_IPARAMS);
+      multis[i] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
     }
-    // The last thread uses ONLY the DefaultFacetIndexingParams so that
-    // it references a different TFC cache. This will still result
-    // in valid results, but will only search one of the category lists
-    // instead of all of them.
-    multis[numThreads - 1] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
 
-    // Gentleman, start your engines
     for (Multi m : multis) {
       m.start();
     }
 
     // Wait for threads and get results
-    ExampleResult[] multiResults = new ExampleResult[numThreads];
+    String[] expLabelsA = new String[] { "a/d", "a/e", "a/b" };
-    for (int i = 0; i < numThreads; i++) {
+    String[] expLabelsB = new String[] { "b/c", "b/a" };
-      multis[i].join();
+    for (Multi m : multis) {
-      multiResults[i] = multis[i].getResults();
+      m.join();
+      List<FacetResult> facetResults = m.getResults();
+      assertEquals("expected two results", 2, facetResults.size());
+      
+      FacetResultNode nodeA = facetResults.get(0).getFacetResultNode();
+      int i = 0;
+      for (FacetResultNode node : nodeA.subResults) {
+        assertEquals("wrong count", 1, (int) node.value);
+        assertEquals(expLabelsA[i++], node.label.toString('/'));
+      }
+      
+      FacetResultNode nodeB = facetResults.get(1).getFacetResultNode();
+      i = 0;
+      for (FacetResultNode node : nodeB.subResults) {
+        assertEquals("wrong count", 1, (int) node.value);
+        assertEquals(expLabelsB[i++], node.label.toString('/'));
+      }
     }
-
+    
-    // Each of the (numThreads-1) should have the same predictable
+    IOUtils.close(slowIndexReader, slowTaxoReader, indexDir, taxoDir);
-    // results, which we test for here.
-    for (int i = 0; i < numThreads - 1; i++) {
-      ExampleResult eResults = multiResults[i];
-      TestMultiCLExample.assertCorrectMultiResults(eResults);
-    }
-
-    // The last thread, which only searched over the
-    // DefaultFacetIndexingParams,
-    // has its own results
-    ExampleResult eResults = multiResults[numThreads - 1];
-    List<FacetResult> results = eResults.getFacetResults();
-    assertEquals(3, results.size());
-    String[] expLabels = new String[] { "5", "5/5", "6/2" };
-    double[] expValues = new double[] { 0.0, 0.0, 1.0 };
-    for (int i = 0; i < 3; i++) {
-      FacetResult result = results.get(i);
-      assertNotNull("Result should not be null", result);
-      FacetResultNode resNode = result.getFacetResultNode();
-      assertEquals("Invalid label", expLabels[i], resNode.label.toString());
-      assertEquals("Invalid value", expValues[i], resNode.value, 0.0);
-      assertEquals("Invalid number of subresults", 0, resNode.subResults.size());
-    }
-    // we're done, close the index reader and the taxonomy.
-    slowIndexReader.close();
-    slowTaxoReader.close();
-    indexDir.close();
-    taxoDir.close();
   }
 
   /**
@@ -245,77 +225,78 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
   @Test
   public void testGenerationalConsistency() throws Exception {
     // Create temporary RAMDirectories
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+    Directory indexDir = newDirectory();
+    Directory taxoDir = newDirectory();
 
     // Create our index/taxonomy writers
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
     FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
 
     // Add a facet to the index
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+    addFacets(iParams, indexWriter, taxoWriter, "a", "b");
 
     // Commit Changes
-    writers[0].indexWriter.commit();
+    indexWriter.commit();
-    writers[0].taxWriter.commit();
+    taxoWriter.commit();
 
     // Open readers
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader = DirectoryReader.open(indexDir);
+    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
 
     // As this is the first time we have invoked the TotalFacetCountsManager, 
     // we should expect to compute and not read from disk.
-    TotalFacetCounts totalCounts = 
+    TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
     int prevGen = assertRecomputed(totalCounts, 0, "after first attempt to get it!");
 
     // Repeating same operation should pull from the cache - not recomputed. 
     assertTrue("Should be obtained from cache at 2nd attempt",totalCounts == 
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+      TFC.getTotalCounts(indexReader, taxoReader, iParams));
 
     // Repeat the same operation as above. but clear first - now should recompute again
     initCache();
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 3rd attempt to get it!");
     
     //store to file
     File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
     initCache();
-    TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.store(outputFile, indexReader, taxoReader, iParams);
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 4th attempt to get it!");
 
     //clear and load
     initCache();
-    TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.load(outputFile, indexReader, taxoReader, iParams);
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertReadFromDisc(totalCounts, prevGen, "after 5th attempt to get it!");
 
     // Add a new facet to the index, commit and refresh readers
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
+    addFacets(iParams, indexWriter, taxoWriter, "c", "d");
-    writers[0].indexWriter.close();
+    IOUtils.close(indexWriter, taxoWriter);
-    writers[0].taxWriter.close();
 
-    DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
+    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
     assertNotNull(newTaxoReader);
-    assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > readers[0].taxReader.getSize());
+    assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > taxoReader.getSize());
-    readers[0].taxReader.close();
+    taxoReader.close();
-    readers[0].taxReader = newTaxoReader;
+    taxoReader = newTaxoReader;
     
-    DirectoryReader r2 = DirectoryReader.openIfChanged(readers[0].indexReader);
+    DirectoryReader r2 = DirectoryReader.openIfChanged(indexReader);
     assertNotNull(r2);
-    readers[0].indexReader.close();
+    indexReader.close();
-    readers[0].indexReader = r2;
+    indexReader = r2;
 
     // now use the new reader - should recompute
-    totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     prevGen = assertRecomputed(totalCounts, prevGen, "after updating the index - 7th attempt!");
 
     // try again - should not recompute
     assertTrue("Should be obtained from cache at 8th attempt",totalCounts == 
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+      TFC.getTotalCounts(indexReader, taxoReader, iParams));
     
-    readers[0].close();
+    IOUtils.close(indexReader, taxoReader);
     outputFile.delete();
-    IOUtils.close(dirs[0]);
+    IOUtils.close(indexDir, taxoDir);
   }
 
   private int assertReadFromDisc(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
@@ -341,10 +322,12 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
    */
   @Test
   public void testGrowingTaxonomy() throws Exception {
-    // Create temporary RAMDirectories
+    Directory indexDir = newDirectory();
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
+    Directory taxoDir = newDirectory();
+    
     // Create our index/taxonomy writers
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
     FacetIndexingParams iParams = new FacetIndexingParams() {
       @Override
       public int getPartitionSize() {
@@ -352,37 +335,38 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
       }
     };
     // Add a facet to the index
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+    addFacets(iParams, indexWriter, taxoWriter, "a", "b");
     // Commit Changes
-    writers[0].indexWriter.commit();
+    indexWriter.commit();
-    writers[0].taxWriter.commit();
+    taxoWriter.commit();
 
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader = DirectoryReader.open(indexDir);
+    TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
 
     // Create TFC and write cache to disk
     File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
-    TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.store(outputFile, indexReader, taxoReader, iParams);
     
     // Make the taxonomy grow without touching the index
     for (int i = 0; i < 10; i++) {
-      writers[0].taxWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
+      taxoWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
     }
-    writers[0].taxWriter.commit();
+    taxoWriter.commit();
-    DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
+    TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
     assertNotNull(newTaxoReader);
-    readers[0].taxReader.close();
+    taxoReader.close();
-    readers[0].taxReader = newTaxoReader;
+    taxoReader = newTaxoReader;
 
     initCache();
 
     // With the bug, this next call should result in an exception
-    TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
+    TFC.load(outputFile, indexReader, taxoReader, iParams);
-    TotalFacetCounts totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
     assertReadFromDisc(totalCounts, 0, "after reading from disk.");
+    
     outputFile.delete();
-    writers[0].close();
+    IOUtils.close(indexWriter, taxoWriter, indexReader, taxoReader);
-    readers[0].close();
+    IOUtils.close(indexDir, taxoDir);
-    IOUtils.close(dirs[0]);
   }
 
   /**
@@ -445,46 +429,52 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
    */
   @Test
   public void testMultipleIndices() throws IOException {
-    // Create temporary RAMDirectories
+    Directory indexDir1 = newDirectory(), indexDir2 = newDirectory();
-    Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(2);
+    Directory taxoDir1 = newDirectory(), taxoDir2 = newDirectory();
+    
     // Create our index/taxonomy writers
-    IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
+    IndexWriter indexWriter1 = new IndexWriter(indexDir1, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    IndexWriter indexWriter2 = new IndexWriter(indexDir2, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
+    TaxonomyWriter taxoWriter1 = new DirectoryTaxonomyWriter(taxoDir1);
+    TaxonomyWriter taxoWriter2 = new DirectoryTaxonomyWriter(taxoDir2);
     FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
 
     // Add a facet to the index
-    addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
+    addFacets(iParams, indexWriter1, taxoWriter1, "a", "b");
-    addFacets(iParams, writers[1].indexWriter, writers[1].taxWriter, "d", "e");
+    addFacets(iParams, indexWriter1, taxoWriter1, "d", "e");
     // Commit Changes
-    writers[0].indexWriter.commit();
+    indexWriter1.commit();
-    writers[0].taxWriter.commit();
+    indexWriter2.commit();
-    writers[1].indexWriter.commit();
+    taxoWriter1.commit();
-    writers[1].taxWriter.commit();
+    taxoWriter2.commit();
 
     // Open two readers
-    IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
+    DirectoryReader indexReader1 = DirectoryReader.open(indexDir1);
+    DirectoryReader indexReader2 = DirectoryReader.open(indexDir2);
+    TaxonomyReader taxoReader1 = new DirectoryTaxonomyReader(taxoDir1);
+    TaxonomyReader taxoReader2 = new DirectoryTaxonomyReader(taxoDir2);
 
     // As this is the first time we have invoked the TotalFacetCountsManager, we
     // should expect to compute.
-    TotalFacetCounts totalCounts0 = 
+    TotalFacetCounts totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
-      TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
     int prevGen = -1;
     prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 1");
     assertTrue("attempt 1b for same input [0] shout find it in cache",
-        totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+        totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
     
     // 2nd Reader - As this is the first time we have invoked the
     // TotalFacetCountsManager, we should expect a state of NEW to be returned.
-    TotalFacetCounts totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+    TotalFacetCounts totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
     prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 2");
     assertTrue("attempt 2b for same input [1] shout find it in cache",
-        totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
+        totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
 
     // Right now cache size is one, so first TFC is gone and should be recomputed  
-    totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
     prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 3");
     
     // Similarly will recompute the second result  
-    totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+    totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
     prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 4");
 
     // Now we set the cache size to two, meaning both should exist in the
@@ -492,23 +482,19 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
     TFC.setCacheSize(2);
 
     // Re-compute totalCounts0 (was evicted from the cache when the cache was smaller)
-    totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
+    totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
     prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 5");
 
     // now both are in the larger cache and should not be recomputed 
-    totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
+    totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
     assertTrue("with cache of size 2 res no. 0 should come from cache",
-        totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
+        totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
     assertTrue("with cache of size 2 res no. 1 should come from cache",
-        totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
+        totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
     
-    writers[0].close();
+    IOUtils.close(indexWriter1, indexWriter2, taxoWriter1, taxoWriter2);
-    writers[1].close();
+    IOUtils.close(indexReader1, indexReader2, taxoReader1, taxoReader2);
-    readers[0].close();
+    IOUtils.close(indexDir1, indexDir2, taxoDir1, taxoDir2);
-    readers[1].close();
-    for (Directory[] dirset : dirs) {
-      IOUtils.close(dirset);
-    }
   }
 
 }

lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java

@@ -3,6 +3,7 @@ package org.apache.lucene.facet.search.sampling;
 import java.util.List;
 import java.util.Random;
 
+import org.apache.lucene.facet.index.params.FacetIndexingParams;
 import org.apache.lucene.facet.search.BaseTestTopK;
 import org.apache.lucene.facet.search.FacetsAccumulator;
 import org.apache.lucene.facet.search.FacetsCollector;
@@ -46,8 +47,8 @@ public abstract class BaseSampleTestTopK extends BaseTestTopK {
   protected static final int RETRIES = 10;
   
   @Override
-  protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
+  protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
-    FacetSearchParams res = super.searchParamsWithRequests(numResults, partitionSize);
+    FacetSearchParams res = super.searchParamsWithRequests(numResults, fip);
     for (FacetRequest req : res.facetRequests) {
       // randomize the way we aggregate results
       if (random().nextBoolean()) {
@@ -71,20 +72,23 @@ public abstract class BaseSampleTestTopK extends BaseTestTopK {
     boolean useRandomSampler = random().nextBoolean();
     for (int partitionSize : partitionSizes) {
       try {
-        initIndex(partitionSize);
+        // complements return counts for all ordinals, so force ALL_PARENTS indexing
+        // so that it's easier to compare
+        FacetIndexingParams fip = getFacetIndexingParams(partitionSize, true);
+        initIndex(fip);
         // Get all of the documents and run the query, then do different
         // facet counts and compare to control
         Query q = new TermQuery(new Term(CONTENT_FIELD, BETA)); // 90% of the docs
         ScoredDocIdCollector docCollector = ScoredDocIdCollector.create(indexReader.maxDoc(), false);
         
-        FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, partitionSize); 
+        FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, fip); 
         FacetsCollector fc = FacetsCollector.create(expectedSearchParams, indexReader, taxoReader);
         
         searcher.search(q, MultiCollector.wrap(docCollector, fc));
         
         List<FacetResult> expectedResults = fc.getFacetResults();
         
-        FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, partitionSize); 
+        FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, fip); 
         
         // try several times in case of failure, because the test has a chance to fail 
         // if the top K facets are not sufficiently common with the sample set

lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java

@@ -3,7 +3,6 @@ package org.apache.lucene.facet.search.sampling;
 import java.io.IOException;
 import java.util.Collections;
 
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.facet.FacetTestCase;
 import org.apache.lucene.facet.index.FacetFields;
@@ -95,7 +94,7 @@ public class OversampleWithDepthTest extends FacetTestCase {
   }
 
   private void index100Docs(Directory indexDir, Directory taxoDir, FacetIndexingParams fip) throws IOException {
-    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer());
+    IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
     IndexWriter w = new IndexWriter(indexDir, iwc);
     TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);
     

lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java

@@ -3,7 +3,7 @@ package org.apache.lucene.facet.taxonomy.directory;
 import java.io.IOException;
 import java.util.Random;
 
-import org.apache.lucene.analysis.core.KeywordAnalyzer;
+import org.apache.lucene.analysis.MockAnalyzer;
 import org.apache.lucene.facet.FacetTestCase;
 import org.apache.lucene.facet.taxonomy.CategoryPath;
 import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@@ -256,7 +256,7 @@ public class TestDirectoryTaxonomyReader extends FacetTestCase {
     // hold onto IW to forceMerge
     // note how we don't close it, since DTW will close it.
     final IndexWriter iw = new IndexWriter(dir,
-        new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
             .setMergePolicy(new LogByteSizeMergePolicy()));
     DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
       @Override
@@ -299,7 +299,7 @@ public class TestDirectoryTaxonomyReader extends FacetTestCase {
     // hold onto IW to forceMerge
     // note how we don't close it, since DTW will close it.
     final IndexWriter iw = new IndexWriter(dir,
-        new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
+        new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
             .setMergePolicy(new LogByteSizeMergePolicy()));
     DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
       @Override

lucene/site/xsl/index.xsl

@@ -58,7 +58,8 @@
         <li><a href="core/overview-summary.html#overview_description">Introduction to Lucene's APIs</a>:
         High-level summary of the different Lucene packages. </li>
         <li><a href="core/org/apache/lucene/analysis/package-summary.html#package_description">Analysis overview</a>:
-        Introduction to Lucene's analysis API. </li>
+        Introduction to Lucene's analysis API.  See also the
+        <a href="core/org/apache/lucene/analysis/TokenStream.html">TokenStream consumer workflow</a>.</li>
         </ul>
         <h2>Reference Documents</h2>
           <ul>

lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java

@@ -19,6 +19,7 @@ package org.apache.lucene.analysis;
 
 import java.io.IOException;
 import java.io.Reader;
+import java.util.Map;
 import java.util.SortedMap;
 import java.util.TreeMap;
 
@@ -89,8 +90,8 @@ public class MockCharFilter extends CharFilter {
 
   @Override
   public int correct(int currentOff) {
-    SortedMap<Integer,Integer> subMap = corrections.subMap(0, currentOff+1);
+    Map.Entry<Integer,Integer> lastEntry = corrections.lowerEntry(currentOff+1);
-    int ret = subMap.isEmpty() ? currentOff : currentOff + subMap.get(subMap.lastKey());
+    int ret = lastEntry == null ? currentOff : currentOff + lastEntry.getValue();
     assert ret >= 0 : "currentOff=" + currentOff + ",diff=" + (ret-currentOff);
     return ret;
   }

solr/CHANGES.txt

@@ -95,6 +95,11 @@ Bug Fixes
 
 * SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
 
+* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
+
+* SOLR-4380: Replicate after startup option would not replicate until the
+  IndexWriter was lazily opened. (Mark Miller, Gregg Donovan)
+
 Optimizations
 ----------------------
 
@@ -115,6 +120,8 @@ Optimizations
 * SOLR-4284: Admin UI - make core list scrollable separate from the rest of
   the UI (steffkes)
 
+* SOLR-4364: Admin UI - Locale based number formatting (steffkes)
+
 Other Changes
 ----------------------
 
@@ -123,6 +130,8 @@ Other Changes
 
 * SOLR-4353: Renamed example jetty context file to reduce confusion (hossman)
 
+* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
+
 ==================  4.1.0 ==================
 
 Versions of Major Components

solr/build.xml

@@ -560,17 +560,8 @@
     <echo message="Checking for broken links..."/>
     <check-broken-links dir="${javadoc.dir}"/>
     <echo message="Checking for malformed docs..."/>
-    <!-- TODO: add missing package.htmls and bump this to level=package -->
+    <!-- TODO: add missing docs for all classes and bump this to level=class -->
-    <check-missing-javadocs dir="${javadoc.dir}" level="none"/>
+    <check-missing-javadocs dir="${javadoc.dir}" level="package"/>
-    <!-- prevent the modules without problems from getting worse -->
-    <check-missing-javadocs dir="${javadoc.dir}/solr-analysis-extras" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-cell" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-clustering" level="package"/>
-    <!-- solr-core: problems: -->
-    <check-missing-javadocs dir="${javadoc.dir}/solr-dataimporthandler" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-dataimporthandler-extras" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-solrj" level="package"/>
-    <check-missing-javadocs dir="${javadoc.dir}/solr-test-framework" level="package"/>
   </target>
  
   <target name="-ecj-javadoc-lint" depends="documentation,compile-solr-test-framework,-ecj-resolve">

solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml

@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a 
      more concise example.
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.0">

solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml

@@ -17,11 +17,6 @@
  limitations under the License.
 -->
 
-<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source$
-     $Name$
-  -->
-
 <config>
   <luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
   <jmx />

solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java

@@ -222,10 +222,6 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro
             log.debug("Mapping field "+fieldName+" using document global language "+fieldLang);
           }
           String mappedOutputField = getMappedField(fieldName, fieldLang);
-          if(enforceSchema && schema.getFieldOrNull(fieldName) == null) {
-            log.warn("Unsuccessful field name mapping to {}, field does not exist, skipping mapping.", mappedOutputField, fieldName);
-            mappedOutputField = fieldName;
-          }
 
           if (mappedOutputField != null) {
             log.debug("Mapping field {} to {}", doc.getFieldValue(docIdField), fieldLang);
@@ -350,17 +346,23 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro
 
   /**
    * Returns the name of the field to map the current contents into, so that they are properly analyzed.  For instance
-   * if the currentField is "text" and the code is "en", the new field would be "text_en".  If such a field doesn't exist,
+   * if the currentField is "text" and the code is "en", the new field would by default be "text_en".
-   * then null is returned.
+   * This method also performs custom regex pattern replace if configured. If enforceSchema=true
+   * and the resulting field name doesn't exist, then null is returned.
    *
    * @param currentField The current field name
    * @param language the language code
-   * @return The new schema field name, based on pattern and replace
+   * @return The new schema field name, based on pattern and replace, or null if illegal
    */
   protected String getMappedField(String currentField, String language) {
     String lc = lcMap.containsKey(language) ? lcMap.get(language) : language;
     String newFieldName = langPattern.matcher(mapPattern.matcher(currentField).replaceFirst(mapReplaceStr)).replaceFirst(lc);
-    log.debug("Doing mapping from "+currentField+" with language "+language+" to field "+newFieldName);
+    if(enforceSchema && schema.getFieldOrNull(newFieldName) == null) {
+      log.warn("Unsuccessful field name mapping from {} to {}, field does not exist and enforceSchema=true; skipping mapping.", currentField, newFieldName);
+      return null;
+    } else {
+      log.debug("Doing mapping from "+currentField+" with language "+language+" to field "+newFieldName);
+    }
     return newFieldName;
   }
 

solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml

@@ -23,9 +23,6 @@
      kitchen sink thrown in. See example/solr/conf/schema.xml for a
      more concise example.
 
-     $Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
-     $Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
-     $Name:  $
   -->
 
 <schema name="test" version="1.5">

solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java

@@ -93,7 +93,7 @@ public abstract class LanguageIdentifierUpdateProcessorFactoryTestCase extends S
     parameters = new ModifiableSolrParams();
     parameters.add("langid.fl", "name");
     parameters.add("langid.map.lcmap", "jp:s zh:cjk ko:cjk");
-    parameters.add("langid.enforceSchema", "true");
+    parameters.set("langid.enforceSchema", "false");
     liProcessor = createLangIdProcessor(parameters);
     
     assertEquals("test_no", liProcessor.getMappedField("test", "no"));
@@ -102,13 +102,17 @@ public abstract class LanguageIdentifierUpdateProcessorFactoryTestCase extends S
     assertEquals("test_cjk", liProcessor.getMappedField("test", "zh"));
     assertEquals("test_cjk", liProcessor.getMappedField("test", "ko"));
 
-    // Prove support for other mapping regex
+    // Test that enforceSchema correctly catches illegal field and returns null
-    parameters.add("langid.map.pattern", "text_(.*?)_field");
+    parameters.set("langid.enforceSchema", "true");
-    parameters.add("langid.map.replace", "$1_{lang}Text");
     liProcessor = createLangIdProcessor(parameters);
+    assertEquals(null, liProcessor.getMappedField("inputfield", "sv"));
 
-    assertEquals("title_noText", liProcessor.getMappedField("text_title_field", "no"));
+    // Prove support for other mapping regex, still with enforceSchema=true
-    assertEquals("body_svText", liProcessor.getMappedField("text_body_field", "sv"));
+    parameters.add("langid.map.pattern", "text_(.*?)_field");
+    parameters.add("langid.map.replace", "$1_{lang}_s");
+    liProcessor = createLangIdProcessor(parameters);
+    assertEquals("title_no_s", liProcessor.getMappedField("text_title_field", "no"));
+    assertEquals("body_sv_s", liProcessor.getMappedField("text_body_field", "sv"));
   }
 
   @Test

solr/core/src/java/org/apache/solr/analysis/package.html

@@ -0,0 +1,30 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Factories and classes specific to text analysis and the creation of {@link org.apache.lucene.analysis.TokenStream}s
+</p>
+<p>
+See {@link org.apache.lucene.analysis} for additional details.
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/client/solrj/embedded/package.html

@@ -0,0 +1,30 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+SolrJ client implementations for embedded solr access.
+</p>
+<p>
+See {@link org.apache.solr.client.solrj} for additional details.
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/cloud/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Classes for dealing with ZooKeeper when operating in <a href="http://wiki.apache.org/solr/SolrCloud">SolrCloud</a> mode.
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/common/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Commonly reused classes and interfaces (deprecated package, do not add new classes)
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/core/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Core classes implementin Solr internals and the management of {@link org.apache.solr.core.SolrCore}s
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java

@@ -42,6 +42,7 @@ import org.apache.commons.io.IOUtils;
 import org.apache.lucene.index.IndexCommit;
 import org.apache.lucene.index.IndexDeletionPolicy;
 import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexInput;
@@ -877,9 +878,9 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
             }
           }
 
-          // reboot the writer on the new index
+          // ensure the writer is init'd so that we have a list of commit points
-          // TODO: perhaps this is no longer necessary then?
+          RefCounted<IndexWriter> iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
-         // core.getUpdateHandler().newIndexWriter(true);
+          iw.decref();
 
         } catch (IOException e) {
           LOG.warn("Unable to get IndexCommit on startup", e);

solr/core/src/java/org/apache/solr/handler/admin/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+{@link org.apache.solr.request.SolrRequestHandler} implementations for powering he Solr Admin UI
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/handler/component/package.html

@@ -0,0 +1,28 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+{@link org.apache.solr.handler.component.SearchComponent} implementations for 
+use in {@link org.apache.solr.handler.component.SearchHandler}
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/handler/loader/package.html

@@ -0,0 +1,28 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+{@link org.apache.solr.handler.loader.ContentStreamLoader} implementations for 
+use in {@link org.apache.solr.handler.ContentStreamHandlerBase} implementations
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/handler/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Concrete implementations of {@link org.apache.solr.request.SolrRequestHandler}
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/highlight/package.html

@@ -0,0 +1,30 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+{@link org.apache.solr.highlight.SolrHighlighter} API and related implementaions and utilities
+</p>
+<p>
+See {@link org.apache.lucene.search.highlight} for addition information.
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/internal/csv/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Internal classes used for reading/writing CSV
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/internal/csv/writer/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Internal classes used for reading/writing CSV
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/logging/jul/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+JUL based implementation of {@link org.apache.solr.logging.LogWatcher}
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/logging/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+APIs related to capturing log event info in the {@link org.apache.solr.handler.admin.LoggingHandler}
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Commonly reused classes and interfaces (deprecated package, do not add new classes)
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/parser/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Solr native variant of the {@linkplain org.apache.lucene.queryparser.classic.QueryParser Lucene Classic QueryParser}
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/request/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+APIs and classes for dealing with Solr requests
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/response/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+API and implementations of {@link org.apache.solr.response.QueryResponseWriter} for formating Solr request responses
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/response/transform/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+APIs and implementations of {@link org.apache.solr.response.transform.DocTransformer} for modifying documents in Solr request responses
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/schema/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+{@link org.apache.solr.schema.IndexSchema} and {@link org.apache.solr.schema.FieldType} implementations for powering schema.xml
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/search/function/distance/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Solr implementations of {@link org.apache.lucene.queries.function.ValueSource} for distance based function queries.
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/search/function/package.html

@@ -0,0 +1,27 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Solr implementations of {@link org.apache.lucene.queries.function.ValueSource} for function queries.
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/search/grouping/collector/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Grouping related {@link org.apache.lucene.search.Collector}s
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/search/grouping/distributed/command/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Internal classes used to implement distributed result grouping
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/search/grouping/distributed/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Internal APIs for distribute result grouping
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Internal classes used to implement distributed result grouping
+
+
+</p>
+</body>
+</html>

solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/package.html

@@ -0,0 +1,29 @@
+<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+<head>
+   <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
+</head>
+<body>
+<p>
+Internal classes used to implement distributed result grouping
+
+
+</p>
+</body>
+</html>