Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merged /lucene/dev/trunk:r1439980-1440837 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene4547@1440839 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2013-01-31 03:37:31 +00:00
parent 3f0d6bab42 7af430e071
commit b21ecede84
161 changed files with 2340 additions and 819 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
dev-tools/idea/lucene/demo/demo.iml
View File

@ -24,6 +24,7 @@
</orderEntry>
<orderEntry type="module" module-name="analysis-common" />
<orderEntry type="module" module-name="lucene-core" />
<orderEntry type="module" module-name="facet" />
<orderEntry type="module" module-name="queryparser" />
</component>
</module>

5
dev-tools/maven/lucene/demo/pom.xml.template
View File

@ -59,6 +59,11 @@
<artifactId>lucene-analyzers-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>lucene-facet</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>lucene-queryparser</artifactId>

8
lucene/CHANGES.txt
View File

@ -61,6 +61,11 @@ Optimizations
* LUCENE-4690: Performance improvements and non-hashing versions
of NumericUtils.*ToPrefixCoded() (yonik)
* LUCENE-4715: CategoryListParams.getOrdinalPolicy now allows to return a
different OrdinalPolicy per dimension, to better tune how you index
facets. Also added OrdinalPolicy.ALL_BUT_DIMENSION.
(Shai Erera, Michael McCandless)
New Features
* LUCENE-4686: New specialized DGapVInt8IntEncoder for facets (now the
@ -115,6 +120,9 @@ Bug Fixes
you have a taxonomy index created with such strings, you should rebuild it.
(Michael McCandless, Shai Erera)
* LUCENE-4732: Fixed TermsEnum.seekCeil/seekExact on term vectors.
(Adrien Grand, Robert Muir)
======================= Lucene 4.1.0 =======================
Changes in backwards compatibility policy

3
lucene/build.xml
View File

@ -191,8 +191,7 @@
<exclude name="analysis/stempel/classes/java/org/egothor/stemmer/Compile.class"/>
<exclude name="analysis/stempel/classes/java/org/egothor/stemmer/DiffIt.class"/>
<exclude name="benchmark/**"/>
<exclude name="demo/classes/java/org/apache/lucene/demo/IndexFiles.class"/>
<exclude name="demo/classes/java/org/apache/lucene/demo/SearchFiles.class"/>
<exclude name="demo/classes/java/org/apache/lucene/**"/>
<exclude name="misc/classes/java/org/apache/lucene/index/CompoundFileExtractor.class"/>
<exclude name="misc/classes/java/org/apache/lucene/index/IndexSplitter.class"/>
<exclude name="misc/classes/java/org/apache/lucene/index/MultiPassIndexSplitter.class"/>

16
lucene/core/src/java/org/apache/lucene/codecs/compressing/CompressingTermVectorsReader.java
View File

@ -832,10 +832,18 @@ public final class CompressingTermVectorsReader extends TermVectorsReader implem
}
}
// linear scan
do {
next();
} while (ord < numTerms - 1 && term().compareTo(text) < 0);
return term().equals(text) ? SeekStatus.FOUND : SeekStatus.END;
while (true) {
final BytesRef term = next();
if (term == null) {
return SeekStatus.END;
}
final int cmp = term.compareTo(text);
if (cmp > 0) {
return SeekStatus.NOT_FOUND;
} else if (cmp == 0) {
return SeekStatus.FOUND;
}
}
}
@Override

2
lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsReader.java
View File

@ -422,7 +422,7 @@ public class Lucene40TermVectorsReader extends TermVectorsReader implements Clos
this.storePayloads = storePayloads;
nextTerm = 0;
tvf.seek(tvfFPStart);
tvfFP = 1+tvfFPStart;
tvfFP = tvfFPStart;
positions = null;
startOffsets = null;
endOffsets = null;

43
lucene/core/src/test/org/apache/lucene/index/TestDuelingCodecs.java
View File

@ -38,6 +38,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.FixedBitSet;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.automaton.AutomatonTestUtil;
import org.apache.lucene.util.automaton.CompiledAutomaton;
import org.apache.lucene.util.automaton.RegExp;
@ -264,36 +265,50 @@ public class TestDuelingCodecs extends LuceneTestCase {
byte newbytes[] = new byte[term.length+5];
System.arraycopy(term.bytes, term.offset, newbytes, 5, term.length);
tests.add(new BytesRef(newbytes, 5, term.length));
} else if (code == 3) {
switch (random().nextInt(3)) {
case 0:
tests.add(new BytesRef()); // before the first term
break;
case 1:
tests.add(new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF})); // past the last term
break;
case 2:
tests.add(new BytesRef(_TestUtil.randomSimpleString(random()))); // random term
break;
default:
throw new AssertionError();
}
}
}
numPasses++;
}
rightEnum = rightTerms.iterator(rightEnum);
ArrayList<BytesRef> shuffledTests = new ArrayList<BytesRef>(tests);
Collections.shuffle(shuffledTests, random);
for (BytesRef b : shuffledTests) {
if (rarely()) {
// reuse the enums
leftEnum = leftTerms.iterator(leftEnum);
rightEnum = rightTerms.iterator(rightEnum);
assertEquals(info, leftEnum.seekExact(b, false), rightEnum.seekExact(b, false));
assertEquals(info, leftEnum.seekExact(b, true), rightEnum.seekExact(b, true));
SeekStatus leftStatus;
SeekStatus rightStatus;
leftStatus = leftEnum.seekCeil(b, false);
rightStatus = rightEnum.seekCeil(b, false);
assertEquals(info, leftStatus, rightStatus);
if (leftStatus != SeekStatus.END) {
assertEquals(info, leftEnum.term(), rightEnum.term());
}
leftStatus = leftEnum.seekCeil(b, true);
rightStatus = rightEnum.seekCeil(b, true);
final boolean useCache = random().nextBoolean();
final boolean seekExact = random().nextBoolean();
if (seekExact) {
assertEquals(info, leftEnum.seekExact(b, useCache), rightEnum.seekExact(b, useCache));
} else {
SeekStatus leftStatus = leftEnum.seekCeil(b, useCache);
SeekStatus rightStatus = rightEnum.seekCeil(b, useCache);
assertEquals(info, leftStatus, rightStatus);
if (leftStatus != SeekStatus.END) {
assertEquals(info, leftEnum.term(), rightEnum.term());
assertTermStats(leftEnum, rightEnum);
}
}
}
}

2
lucene/core/src/test/org/apache/lucene/index/TestPayloadsOnVectors.java
View File

@ -251,7 +251,7 @@ public class TestPayloadsOnVectors extends LuceneTestCase {
for (int i = 0; i < len; ++i) {
terms[i] = RandomPicks.randomFrom(random(), sampleTerms);
if (weird) {
positionsIncrements[i] = random().nextInt(1 << 18);
positionsIncrements[i] = _TestUtil.nextInt(random(), 1, 1 << 18);
startOffsets[i] = random().nextInt();
endOffsets[i] = random().nextInt();
} else if (i == 0) {

7
lucene/demo/build.xml
View File

@ -33,20 +33,22 @@
<pathelement path="${analyzers-common.jar}"/>
<pathelement path="${queryparser.jar}"/>
<pathelement path="${lucene-core.jar}"/>
<pathelement path="${facet.jar}"/>
<fileset dir="lib"/>
</path>
<target name="javadocs" depends="javadocs-analyzers-common,javadocs-queryparser,compile-core">
<target name="javadocs" depends="javadocs-analyzers-common,javadocs-queryparser,javadocs-facet,compile-core">
<!-- we link the example source in the javadocs, as its ref'ed elsewhere -->
<invoke-module-javadoc linksource="yes">
<links>
<link href="../analyzers-common"/>
<link href="../queryparser"/>
<link href="../facet"/>
</links>
</invoke-module-javadoc>
</target>
<target name="compile-core" depends="jar-analyzers-common,jar-queryparser,common.compile-core" />
<target name="compile-core" depends="jar-analyzers-common,jar-queryparser,jar-facet,common.compile-core" />
<target name="default" depends="jar-core,build-web-demo"/>
@ -63,6 +65,7 @@
<lib file="${queries.jar}"/>
<lib file="${queryparser.jar}"/>
<lib file="${lucene-core.jar}"/>
<lib file="${facet.jar}"/>
</war>
</target>

9
lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleResult.java → lucene/demo/src/java/org/apache/lucene/demo/facet/ExampleResult.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example;
package org.apache.lucene.demo.facet;
import java.util.List;
@ -30,17 +30,20 @@ import org.apache.lucene.facet.search.results.FacetResult;
*/
public class ExampleResult {
/** Sole constructor. */
public ExampleResult() {}
private List<FacetResult> facetResults;
/**
* @return the facet results
* Returns the facet results
*/
public List<FacetResult> getFacetResults() {
return facetResults;
}
/**
* @param facetResults the facet results to set
* Sets the facet results
*/
public void setFacetResults(List<FacetResult> facetResults) {
this.facetResults = facetResults;

15
lucene/facet/src/examples/org/apache/lucene/facet/example/ExampleUtils.java → lucene/demo/src/java/org/apache/lucene/demo/facet/ExampleUtils.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example;
package org.apache.lucene.demo.facet;
import org.apache.lucene.util.Version;
@ -20,15 +20,28 @@ import org.apache.lucene.util.Version;
*/
/**
* Simple utility functions for the faceting examples
* @lucene.experimental
*/
public class ExampleUtils {
/** No instance */
private ExampleUtils() {}
/**
* True if the system property <code>tests.verbose</code> has been set.
* If true, it causes {@link #log(Object)} to print messages to the console.
*/
public static final boolean VERBOSE = Boolean.getBoolean("tests.verbose");
/** The Lucene {@link Version} used by the example code. */
public static final Version EXAMPLE_VER = Version.LUCENE_40;
/**
* Logs the String representation of <code>msg</code> to the console,
* if {@link #VERBOSE} is true. Otherwise, does nothing.
* @see #VERBOSE
*/
public static void log(Object msg) {
if (VERBOSE) {
System.out.println(msg.toString());

14
lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveMain.java → lucene/demo/src/java/org/apache/lucene/demo/facet/adaptive/AdaptiveMain.java
View File

@ -1,14 +1,14 @@
package org.apache.lucene.facet.example.adaptive;
package org.apache.lucene.demo.facet.adaptive;
import java.util.List;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.example.simple.SimpleIndexer;
import org.apache.lucene.facet.example.simple.SimpleSearcher;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.demo.facet.simple.SimpleIndexer;
import org.apache.lucene.demo.facet.simple.SimpleSearcher;
import org.apache.lucene.facet.search.AdaptiveFacetsAccumulator;
import org.apache.lucene.facet.search.results.FacetResult;
@ -37,6 +37,9 @@ import org.apache.lucene.facet.search.results.FacetResult;
*/
public class AdaptiveMain {
/** Sole constructor */
public AdaptiveMain() {}
/**
* Driver for the adaptive sample.
* @throws Exception on error (no detailed exception handling here for sample simplicity
@ -46,6 +49,7 @@ public class AdaptiveMain {
ExampleUtils.log("DONE");
}
/** Runs the adaptive sample and returns the facet results */
public ExampleResult runSample() throws Exception {
// create Directories for the search index and for the taxonomy index

9
lucene/facet/src/examples/org/apache/lucene/facet/example/adaptive/AdaptiveSearcher.java → lucene/demo/src/java/org/apache/lucene/demo/facet/adaptive/AdaptiveSearcher.java
View File

@ -1,9 +1,9 @@
package org.apache.lucene.facet.example.adaptive;
package org.apache.lucene.demo.facet.adaptive;
import java.util.List;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.example.simple.SimpleUtils;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.demo.facet.simple.SimpleUtils;
import org.apache.lucene.facet.search.AdaptiveFacetsAccumulator;
import org.apache.lucene.facet.search.ScoredDocIdCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
@ -46,6 +46,9 @@ import org.apache.lucene.store.Directory;
*/
public class AdaptiveSearcher {
/** No instance */
private AdaptiveSearcher() {}
/**
* Search with facets through the {@link AdaptiveFacetsAccumulator}
* @param indexDir Directory of the search index.

22
lucene/demo/src/java/org/apache/lucene/demo/facet/adaptive/package.html Normal file
View File

@ -0,0 +1,22 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html><head></head>
<body>
Facets example code for using AdaptiveFacetsAccumulator.
</body>
</html>

9
lucene/facet/src/examples/org/apache/lucene/facet/example/association/CategoryAssociationsIndexer.java → lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsIndexer.java
View File

@ -1,13 +1,13 @@
package org.apache.lucene.facet.example.association;
package org.apache.lucene.demo.facet.association;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.demo.facet.simple.SimpleUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.associations.AssociationsFacetFields;
import org.apache.lucene.facet.associations.CategoryAssociation;
import org.apache.lucene.facet.associations.CategoryAssociationsContainer;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.example.simple.SimpleUtils;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
@ -41,6 +41,9 @@ import org.apache.lucene.store.Directory;
*/
public class CategoryAssociationsIndexer {
/** No instance. */
private CategoryAssociationsIndexer() {}
/**
* Create an index, and adds to it sample documents and categories.
*

15
lucene/facet/src/examples/org/apache/lucene/facet/example/association/CategoryAssociationsMain.java → lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsMain.java
View File

@ -1,12 +1,12 @@
package org.apache.lucene.facet.example.association;
package org.apache.lucene.demo.facet.association;
import java.util.List;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.facet.search.results.FacetResult;
/*
@ -33,6 +33,9 @@ import org.apache.lucene.facet.search.results.FacetResult;
*/
public class CategoryAssociationsMain {
/** Sole constructor. */
public CategoryAssociationsMain() {}
/**
* Driver for the simple sample.
* @throws Exception on error (no detailed exception handling here for sample simplicity
@ -43,6 +46,9 @@ public class CategoryAssociationsMain {
ExampleUtils.log("DONE");
}
/**
* Runs the example demonstrating sum of int-association.
*/
public ExampleResult runSumIntAssociationSample() throws Exception {
// create Directories for the search index and for the taxonomy index
@ -61,6 +67,9 @@ public class CategoryAssociationsMain {
return res;
}
/**
* Runs the example demonstrating sum of float-association.
*/
public ExampleResult runSumFloatAssociationSample() throws Exception {
// create Directories for the search index and for the taxonomy index

7
lucene/facet/src/examples/org/apache/lucene/facet/example/association/CategoryAssociationsSearcher.java → lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsSearcher.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example.association;
package org.apache.lucene.demo.facet.association;
import java.util.List;
@ -6,7 +6,7 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.facet.example.simple.SimpleSearcher;
import org.apache.lucene.demo.facet.simple.SimpleSearcher;
import org.apache.lucene.facet.search.params.associations.AssociationFloatSumFacetRequest;
import org.apache.lucene.facet.search.params.associations.AssociationIntSumFacetRequest;
import org.apache.lucene.facet.search.results.FacetResult;
@ -39,6 +39,9 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
*/
public class CategoryAssociationsSearcher {
/** No instantiation */
private CategoryAssociationsSearcher() {}
/** Search an index with a sum of int-association. */
public static List<FacetResult> searchSumIntAssociation(Directory indexDir, Directory taxoDir) throws Exception {
// prepare index reader

9
lucene/facet/src/examples/org/apache/lucene/facet/example/association/CategoryAssociationsUtils.java → lucene/demo/src/java/org/apache/lucene/demo/facet/association/CategoryAssociationsUtils.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example.association;
package org.apache.lucene.demo.facet.association;
import org.apache.lucene.facet.associations.CategoryAssociation;
import org.apache.lucene.facet.associations.CategoryFloatAssociation;
@ -23,10 +23,14 @@ import org.apache.lucene.facet.taxonomy.CategoryPath;
*/
/**
* Categories for the facet examples
* @lucene.experimental
*/
public class CategoryAssociationsUtils {
/** No instance */
private CategoryAssociationsUtils() {}
/**
* Categories: categories[D][N] == category-path with association no. N for
* document no. D.
@ -45,6 +49,9 @@ public class CategoryAssociationsUtils {
}
};
/**
* Associations (occurrences/confidence levels) for {@link #categories}
*/
public static CategoryAssociation[][] associations = {
// Doc #1 associations
{

22
lucene/demo/src/java/org/apache/lucene/demo/facet/association/package.html Normal file
View File

@ -0,0 +1,22 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html><head></head>
<body>
Facets example code for using associations.
</body>
</html>

34
lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLIndexer.java → lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/MultiCLIndexer.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example.multiCL;
package org.apache.lucene.demo.facet.multiCL;
import java.util.Arrays;
import java.util.HashMap;
@ -6,11 +6,11 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.demo.facet.simple.SimpleUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.example.simple.SimpleUtils;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
@ -49,13 +49,16 @@ import org.apache.lucene.store.RAMDirectory;
*/
public class MultiCLIndexer {
// Number of documents to index
/** No instance */
private MultiCLIndexer() {}
/** Number of documents to index */
public static int NUM_DOCS = 100;
// Number of facets to add per document
/** Number of facets to add per document */
public static int NUM_FACETS_PER_DOC = 10;
// Number of tokens in title
/** Number of tokens in title */
public static int TITLE_LENGTH = 5;
// Number of tokens in text
/** Number of tokens in text */
public static int TEXT_LENGTH = 100;
// Lorum ipsum to use as content - this will be tokenized and used for document
@ -73,7 +76,7 @@ public class MultiCLIndexer {
+ "nisi ut aliquid ex ea commodi consequatur? Quis autem vel eum iure"
+ "reprehenderit qui in ea voluptate velit esse quam nihil molestiae "
+ "consequatur vel illum qui dolorem eum fugiat quo voluptas nulla pariatur";
// PerDimensionIndexingParams for multiple category lists
/** PerDimensionIndexingParams for multiple category lists */
public static final PerDimensionIndexingParams MULTI_IPARAMS;
// Initialize PerDimensionIndexingParams
@ -90,12 +93,16 @@ public class MultiCLIndexer {
/**
* Create an index, and adds to it sample documents and facets.
* @param indexDir Directory in which the index should be created.
* @param taxoDir Directory in which the taxonomy index should be created.
* @throws Exception on error (no detailed exception handling here for sample simplicity
*
* @param indexDir
* Directory in which the index should be created.
* @param taxoDir
* Directory in which the taxonomy index should be created.
* @throws Exception
* on error (no detailed exception handling here for sample
* simplicity
*/
public static void index(Directory indexDir, Directory taxoDir)
throws Exception {
public static void index(Directory indexDir, Directory taxoDir) throws Exception {
Random random = new Random(2003);
@ -195,6 +202,7 @@ public class MultiCLIndexer {
+ nFacetsAdded + " facets.");
}
/** Driver for the example */
public static void main(String[] args) throws Exception {
index(new RAMDirectory(), new RAMDirectory());
}

14
lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLMain.java → lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/MultiCLMain.java
View File

@ -1,12 +1,12 @@
package org.apache.lucene.facet.example.multiCL;
package org.apache.lucene.demo.facet.multiCL;
import java.util.List;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.facet.search.results.FacetResult;
/*
@ -27,12 +27,17 @@ import org.apache.lucene.facet.search.results.FacetResult;
*/
/**
* Driver for the multi sample.
*
* @lucene.experimental
*/
public class MultiCLMain {
/** Sole constructor. */
public MultiCLMain() {}
/**
* Driver for the multi sample.
* Executes the multi sample.
*
* @throws Exception
* on error (no detailed exception handling here for sample
@ -43,6 +48,7 @@ public class MultiCLMain {
ExampleUtils.log("DONE");
}
/** Runs the multi sample and returns the facet results */
public ExampleResult runSample() throws Exception {
// create Directories for the search index and for the taxonomy index

21
lucene/facet/src/examples/org/apache/lucene/facet/example/multiCL/MultiCLSearcher.java → lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/MultiCLSearcher.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example.multiCL;
package org.apache.lucene.demo.facet.multiCL;
import java.util.ArrayList;
import java.util.List;
@ -13,8 +13,8 @@ import org.apache.lucene.search.TopScoreDocCollector;
import org.apache.lucene.store.Directory;
import org.apache.lucene.search.MultiCollector;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.example.simple.SimpleUtils;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.demo.facet.simple.SimpleUtils;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
@ -50,6 +50,9 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
*/
public class MultiCLSearcher {
/** No instance */
private MultiCLSearcher() {}
/**
* Search an index with facets.
*
@ -78,6 +81,18 @@ public class MultiCLSearcher {
return results;
}
/**
* Search an index with facets.
*
* @param indexReader
* Reader over the search index.
* @param taxo
* taxonomy reader.
* @throws Exception
* on error (no detailed exception handling here for sample
* simplicity
* @return facet results
*/
public static List<FacetResult> searchWithFacets(IndexReader indexReader,
TaxonomyReader taxo, FacetIndexingParams iParams) throws Exception {
// prepare searcher to search against

22
lucene/demo/src/java/org/apache/lucene/demo/facet/multiCL/package.html Normal file
View File

@ -0,0 +1,22 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html><head></head>
<body>
Facets example code for using multiple category lists.
</body>
</html>

22
lucene/demo/src/java/org/apache/lucene/demo/facet/package.html Normal file
View File

@ -0,0 +1,22 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html><head></head>
<body>
Facets example code.
</body>
</html>

7
lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleIndexer.java → lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleIndexer.java
View File

@ -1,12 +1,12 @@
package org.apache.lucene.facet.example.simple;
package org.apache.lucene.demo.facet.simple;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
@ -40,6 +40,9 @@ import org.apache.lucene.store.Directory;
*/
public class SimpleIndexer {
/** No instance */
private SimpleIndexer() {}
/**
* Create an index, and adds to it sample documents and facets.
* @param indexDir Directory in which the index should be created.

11
lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleMain.java → lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleMain.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example.simple;
package org.apache.lucene.demo.facet.simple;
import java.util.List;
@ -7,8 +7,8 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
@ -37,6 +37,9 @@ import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
*/
public class SimpleMain {
/** Sole constructor */
public SimpleMain() {}
/**
* Driver for the simple sample.
* @throws Exception on error (no detailed exception handling here for sample simplicity
@ -47,6 +50,7 @@ public class SimpleMain {
ExampleUtils.log("DONE");
}
/** Runs the simple sample and returns the facet results */
public ExampleResult runSimple() throws Exception {
// create Directories for the search index and for the taxonomy index
Directory indexDir = new RAMDirectory();
@ -72,6 +76,7 @@ public class SimpleMain {
return res;
}
/** Runs the simple sample and returns drilldown results */
public ExampleResult runDrillDown() throws Exception {
// create Directories for the search index and for the taxonomy index
Directory indexDir = new RAMDirectory();

7
lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleSearcher.java → lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleSearcher.java
View File

@ -1,9 +1,9 @@
package org.apache.lucene.facet.example.simple;
package org.apache.lucene.demo.facet.simple;
import java.util.Iterator;
import java.util.List;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.DrillDown;
import org.apache.lucene.facet.search.FacetsCollector;
@ -47,6 +47,9 @@ import org.apache.lucene.search.TopScoreDocCollector;
*/
public class SimpleSearcher {
/** No instance */
private SimpleSearcher() {}
/**
* Search an index with facets.
* @param indexReader index reader.

7
lucene/facet/src/examples/org/apache/lucene/facet/example/simple/SimpleUtils.java → lucene/demo/src/java/org/apache/lucene/demo/facet/simple/SimpleUtils.java
View File

@ -1,8 +1,8 @@
package org.apache.lucene.facet.example.simple;
package org.apache.lucene.demo.facet.simple;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.demo.facet.ExampleUtils;
import org.apache.lucene.facet.taxonomy.CategoryPath;
/*
@ -29,6 +29,9 @@ import org.apache.lucene.facet.taxonomy.CategoryPath;
*/
public class SimpleUtils {
/** No instance */
private SimpleUtils() {}
/**
* Documents text field.
*/

22
lucene/demo/src/java/org/apache/lucene/demo/facet/simple/package.html Normal file
View File

@ -0,0 +1,22 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html><head></head>
<body>
Facets simple example code.
</body>
</html>

6
lucene/facet/src/test/org/apache/lucene/facet/example/TestAdaptiveExample.java → lucene/demo/src/test/org/apache/lucene/demo/facet/TestAdaptiveExample.java
View File

@ -1,10 +1,10 @@
package org.apache.lucene.facet.example;
package org.apache.lucene.demo.facet;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.adaptive.AdaptiveMain;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.adaptive.AdaptiveMain;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more

6
lucene/facet/src/test/org/apache/lucene/facet/example/TestAssociationExample.java → lucene/demo/src/test/org/apache/lucene/demo/facet/TestAssociationExample.java
View File

@ -1,10 +1,10 @@
package org.apache.lucene.facet.example;
package org.apache.lucene.demo.facet;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.association.CategoryAssociationsMain;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.association.CategoryAssociationsMain;
import org.apache.lucene.facet.search.results.FacetResultNode;
/*

6
lucene/facet/src/test/org/apache/lucene/facet/example/TestMultiCLExample.java → lucene/demo/src/test/org/apache/lucene/demo/facet/TestMultiCLExample.java
View File

@ -1,4 +1,4 @@
package org.apache.lucene.facet.example;
package org.apache.lucene.demo.facet;
import java.util.Iterator;
import java.util.List;
@ -6,7 +6,8 @@ import java.util.List;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.facet.example.multiCL.MultiCLMain;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.multiCL.MultiCLMain;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
@ -45,7 +46,6 @@ public class TestMultiCLExample extends LuceneTestCase {
assertNotNull("Result should not be null", result);
FacetResultNode node = result.getFacetResultNode();
assertEquals("Invalid label", "5", node.label.toString());
assertEquals("Invalid value", 2.0, node.value, 0.0);
assertEquals("Invalid # of subresults", 3, node.subResults.size());
Iterator<? extends FacetResultNode> subResults = node.subResults.iterator();

6
lucene/facet/src/test/org/apache/lucene/facet/example/TestSimpleExample.java → lucene/demo/src/test/org/apache/lucene/demo/facet/TestSimpleExample.java
View File

@ -1,12 +1,12 @@
package org.apache.lucene.facet.example;
package org.apache.lucene.demo.facet;
import java.util.Iterator;
import org.junit.Test;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.simple.SimpleMain;
import org.apache.lucene.demo.facet.ExampleResult;
import org.apache.lucene.demo.facet.simple.SimpleMain;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;

46
lucene/facet/build.xml
View File

@ -28,52 +28,6 @@
<import file="../module-build.xml"/>
<property name="examples.dir" location="src/examples"/>
<path id="examples.classpath">
<path refid="classpath" />
<pathelement location="${build.dir}/classes/java" />
<pathelement path="${analyzers-common.jar}" />
</path>
<path id="test.classpath">
<path refid="test.base.classpath" />
<pathelement location="${build.dir}/classes/examples" />
</path>
<path id="classpath">
<!-- TODO, cut over tests to MockAnalyzer etc and nuke this dependency -->
<pathelement path="${analyzers-common.jar}" />
<path refid="base.classpath"/>
</path>
<target name="compile-examples" description="Compiles Facets examples">
<compile srcdir="${examples.dir}" destdir="${build.dir}/classes/examples">
<classpath refid="examples.classpath" />
</compile>
</target>
<target name="jar-examples" depends="compile-examples">
<jarify basedir="${build.dir}/classes/examples"
destfile="${build.dir}/${final.name}-examples.jar"
title="Lucene Search Engine: ${ant.project.name}-examples"
manifest.file="${build.dir}/EXAMPLES-MANIFEST.MF">
<fileset dir="src/examples" />
</jarify>
</target>
<target name="compile-core" depends="jar-analyzers-common,common.compile-core,compile-examples" description="Compiles facet classes" />
<target name="jar-core" depends="common.jar-core,jar-examples" />
<target name="javadocs" depends="javadocs-analyzers-common,compile-core">
<invoke-module-javadoc>
<links>
<link href="../analyzers-common"/>
</links>
</invoke-module-javadoc>
</target>
<target name="run-encoding-benchmark" depends="compile-test">
<java classname="org.apache.lucene.util.encoding.EncodingSpeed" fork="true" failonerror="true">
<classpath refid="test.classpath" />

17
lucene/facet/src/examples/org/apache/lucene/facet/example/simple/package.html
View File

@ -1,17 +0,0 @@
<html>
<head>
<title>Simple faceted indexing and search sample</title>
</head>
<body>
<h1>Simple faceted indexing and search sample</h1>
A simple faceted example, showing how to:
<ol>
<li>Create an index.</li>
<li>Add documents with facets to the index.</li>
<li>Search the index.</li>
</ol>
For more complex examples see the other sample code packages.
</body>
</html>

16
lucene/facet/src/java/org/apache/lucene/facet/index/CountingListBuilder.java
View File

@ -3,6 +3,7 @@ package org.apache.lucene.facet.index;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Map;
import java.util.Map.Entry;
@ -115,12 +116,12 @@ public class CountingListBuilder implements CategoryListBuilder {
private final OrdinalsEncoder ordinalsEncoder;
private final TaxonomyWriter taxoWriter;
private final OrdinalPolicy ordinalPolicy;
private final CategoryListParams clp;
public CountingListBuilder(CategoryListParams categoryListParams, FacetIndexingParams indexingParams,
TaxonomyWriter taxoWriter) {
this.taxoWriter = taxoWriter;
this.ordinalPolicy = categoryListParams.getOrdinalPolicy();
this.clp = categoryListParams;
if (indexingParams.getPartitionSize() == Integer.MAX_VALUE) {
ordinalsEncoder = new NoPartitionsOrdinalsEncoder(categoryListParams);
} else {
@ -141,16 +142,23 @@ public class CountingListBuilder implements CategoryListBuilder {
*/
@Override
public Map<String,BytesRef> build(IntsRef ordinals, Iterable<CategoryPath> categories) throws IOException {
int upto = ordinals.length; // since we add ordinals to IntsRef, iterate upto original length
int upto = ordinals.length; // since we may add ordinals to IntsRef, iterate upto original length
if (ordinalPolicy == OrdinalPolicy.ALL_PARENTS) { // add all parents too
Iterator<CategoryPath> iter = categories.iterator();
for (int i = 0; i < upto; i++) {
int ordinal = ordinals.ints[i];
CategoryPath cp = iter.next();
OrdinalPolicy op = clp.getOrdinalPolicy(cp.components[0]);
if (op != OrdinalPolicy.NO_PARENTS) {
// need to add parents too
int parent = taxoWriter.getParent(ordinal);
while (parent > 0) {
ordinals.ints[ordinals.length++] = parent;
parent = taxoWriter.getParent(parent);
}
if (op == OrdinalPolicy.ALL_BUT_DIMENSION) { // discard the last added parent, which is the dimension
ordinals.length--;
}
}
}
return ordinalsEncoder.encode(ordinals);

73
lucene/facet/src/java/org/apache/lucene/facet/index/params/CategoryListParams.java
View File

@ -4,6 +4,7 @@ import java.io.IOException;
import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.DocValuesCategoryListIterator;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.util.PartitionsUtils;
import org.apache.lucene.util.encoding.DGapVInt8IntEncoder;
import org.apache.lucene.util.encoding.IntDecoder;
@ -35,25 +36,61 @@ import org.apache.lucene.util.encoding.UniqueValuesIntEncoder;
*/
public class CategoryListParams {
/** OrdinalPolicy defines which ordinals are encoded for every document. */
/**
* Defines which category ordinals are encoded for every document. This also
* affects how category ordinals are aggregated, check the different policies
* for more details.
*/
public static enum OrdinalPolicy {
/**
* Encodes only the ordinal of leaf nodes. That is, the category A/B/C will
* not encode the ordinals of A and A/B.
* Encodes only the ordinals of leaf nodes. That is, for the category A/B/C,
* the ordinals of A and A/B will not be encoded. This policy is efficient
* for hierarchical dimensions, as it reduces the number of ordinals that
* are visited per document. During faceted search, this policy behaves
* exactly like {@link #ALL_PARENTS}, and the counts of all path components
* will be computed as well.
*
* <p>
* <b>NOTE:</b> this {@link OrdinalPolicy} requires a special collector or
* accumulator, which will fix the parents' counts, unless you are not
* interested in the parents counts.
* accumulator, which will fix the parents' counts.
*
* <p>
* <b>NOTE:</b> since only leaf nodes are encoded for the document, you
* should use this policy when the same document doesn't share two
* categories that have a mutual parent, or otherwise the counts will be
* wrong (the mutual parent will be over-counted). For example, if a
* document has the categories A/B/C and A/B/D, then with this policy the
* counts of "A" and "B" will be 2, which is wrong. If you intend to index
* hierarchical dimensions, with more than one category per document, you
* should use either {@link #ALL_PARENTS} or {@link #ALL_BUT_DIMENSION}.
*/
NO_PARENTS,
/**
* Encodes the ordinals of all path components. That is, the category A/B/C
* will encode the ordinals of A and A/B as well. This is the default
* {@link OrdinalPolicy}.
* will encode the ordinals of A and A/B as well. If you don't require the
* dimension's count during search, consider using
* {@link #ALL_BUT_DIMENSION}.
*/
ALL_PARENTS
ALL_PARENTS,
/**
* Encodes the ordinals of all path components except the dimension. The
* dimension of a category is defined to be the first components in
* {@link CategoryPath#components}. For the category A/B/C, the ordinal of
* A/B will be encoded as well, however not the ordinal of A.
*
* <p>
* <b>NOTE:</b> when facets are aggregated, this policy behaves exactly like
* {@link #ALL_PARENTS}, except that the dimension is never counted. I.e. if
* you ask to count the facet "A", then while in {@link #ALL_PARENTS} you
* will get counts for "A" <u>and its children</u>, with this policy you
* will get counts for <u>only its children</u>. This policy is the default
* one, and makes sense for using with flat dimensions, whenever your
* application does not require the dimension's count. Otherwise, use
* {@link #ALL_PARENTS}.
*/
ALL_BUT_DIMENSION
}
/** The default field used to store the facets information. */
@ -63,7 +100,7 @@ public class CategoryListParams {
* The default {@link OrdinalPolicy} that's used when encoding a document's
* category ordinals.
*/
public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_PARENTS;
public static final OrdinalPolicy DEFAULT_ORDINAL_POLICY = OrdinalPolicy.ALL_BUT_DIMENSION;
public final String field;
@ -115,19 +152,15 @@ public class CategoryListParams {
return false;
}
CategoryListParams other = (CategoryListParams) o;
if (this.hashCode != other.hashCode) {
if (hashCode != other.hashCode) {
return false;
}
// The above hashcodes might equal each other in the case of a collision,
// so at this point only directly term equality testing will settle
// the equality test.
return field.equals(other.field);
}
@Override
public int hashCode() {
return this.hashCode;
return hashCode;
}
/** Create the {@link CategoryListIterator} for the specified partition. */
@ -137,14 +170,18 @@ public class CategoryListParams {
return new DocValuesCategoryListIterator(docValuesField, createEncoder().createMatchingDecoder());
}
/** Returns the {@link OrdinalPolicy} to use for this {@link CategoryListParams}. */
public OrdinalPolicy getOrdinalPolicy() {
/**
* Returns the {@link OrdinalPolicy} to use for the given dimension. This
* {@link CategoryListParams} always returns {@link #DEFAULT_ORDINAL_POLICY}
* for all dimensions.
*/
public OrdinalPolicy getOrdinalPolicy(String dimension) {
return DEFAULT_ORDINAL_POLICY;
}
@Override
public String toString() {
return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy();
return "field=" + field + " encoder=" + createEncoder() + " ordinalPolicy=" + getOrdinalPolicy(null);
}
}

55
lucene/facet/src/java/org/apache/lucene/facet/index/params/PerDimensionOrdinalPolicy.java Normal file
View File

@ -0,0 +1,55 @@
package org.apache.lucene.facet.index.params;
import java.util.Map;
import org.apache.lucene.facet.taxonomy.CategoryPath;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* A {@link CategoryListParams} which allow controlling the
* {@link CategoryListParams.OrdinalPolicy} used for each dimension. The
* dimension is specified as the first component in
* {@link CategoryPath#components}.
*/
public class PerDimensionOrdinalPolicy extends CategoryListParams {
private final Map<String,OrdinalPolicy> policies;
private final OrdinalPolicy defaultOP;
public PerDimensionOrdinalPolicy(Map<String,OrdinalPolicy> policies) {
this(policies, DEFAULT_ORDINAL_POLICY);
}
public PerDimensionOrdinalPolicy(Map<String,OrdinalPolicy> policies, OrdinalPolicy defaultOP) {
this.defaultOP = defaultOP;
this.policies = policies;
}
@Override
public OrdinalPolicy getOrdinalPolicy(String dimension) {
OrdinalPolicy op = policies.get(dimension);
return op == null ? defaultOP : op;
}
@Override
public String toString() {
return super.toString() + " policies=" + policies;
}
}

44
lucene/facet/src/java/org/apache/lucene/facet/search/CountingFacetsCollector.java
View File

@ -81,7 +81,7 @@ import org.apache.lucene.util.encoding.DGapVInt8IntDecoder;
public class CountingFacetsCollector extends FacetsCollector {
private final FacetSearchParams fsp;
private final OrdinalPolicy ordinalPolicy;
private final CategoryListParams clp;
private final TaxonomyReader taxoReader;
private final BytesRef buf = new BytesRef(32);
private final FacetArrays facetArrays;
@ -101,8 +101,7 @@ public class CountingFacetsCollector extends FacetsCollector {
assert assertParams(fsp) == null : assertParams(fsp);
this.fsp = fsp;
CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
this.ordinalPolicy = clp.getOrdinalPolicy();
this.clp = fsp.indexingParams.getCategoryListParams(fsp.facetRequests.get(0).categoryPath);
this.facetsField = clp.field;
this.taxoReader = taxoReader;
this.facetArrays = facetArrays;
@ -209,21 +208,21 @@ public class CountingFacetsCollector extends FacetsCollector {
}
}
private void countParents(int[] parents) {
// counts[0] is the count of ROOT, which we don't care about and counts[1]
// can only update counts[0], so we don't bother to visit it too. also,
// since parents always have lower ordinals than their children, we traverse
// the array backwards. this also allows us to update just the immediate
// parent's count (actually, otherwise it would be a mistake).
for (int i = counts.length - 1; i > 1; i--) {
int count = counts[i];
if (count > 0) {
int parent = parents[i];
if (parent != 0) {
counts[parent] += count;
}
}
/**
* Computes the counts of ordinals under the given ordinal's tree, by
* recursively going down to leaf nodes and rollin up their counts (called
* only with categories are indexing with OrdinalPolicy.NO_PARENTS).
*/
private int rollupCounts(int ordinal, int[] children, int[] siblings) {
int count = 0;
while (ordinal != TaxonomyReader.INVALID_ORDINAL) {
int childCount = counts[ordinal];
childCount += rollupCounts(children[ordinal], children, siblings);
counts[ordinal] = childCount;
count += childCount;
ordinal = siblings[ordinal];
}
return count;
}
@Override
@ -234,11 +233,6 @@ public class CountingFacetsCollector extends FacetsCollector {
ParallelTaxonomyArrays arrays = taxoReader.getParallelTaxonomyArrays();
if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
// need to count parents
countParents(arrays.parents());
}
// compute top-K
final int[] children = arrays.children();
final int[] siblings = arrays.siblings();
@ -248,6 +242,12 @@ public class CountingFacetsCollector extends FacetsCollector {
if (rootOrd == TaxonomyReader.INVALID_ORDINAL) { // category does not exist
continue;
}
OrdinalPolicy ordinalPolicy = clp.getOrdinalPolicy(fr.categoryPath.components[0]);
if (ordinalPolicy == OrdinalPolicy.NO_PARENTS) {
// need to count parents
counts[rootOrd] += rollupCounts(children[rootOrd], children, siblings);
}
FacetResultNode root = new FacetResultNode();
root.ordinal = rootOrd;
root.label = fr.categoryPath;

8
lucene/facet/src/java/org/apache/lucene/facet/search/FacetsCollector.java
View File

@ -43,15 +43,23 @@ public abstract class FacetsCollector extends Collector {
* Returns the most optimized {@link FacetsCollector} for the given search
* parameters. The returned {@link FacetsCollector} is guaranteed to satisfy
* the requested parameters.
*
* @throws IllegalArgumentException
* if there is no built-in collector that can satisfy the search
* parameters.
*/
public static FacetsCollector create(FacetSearchParams fsp, IndexReader indexReader, TaxonomyReader taxoReader) {
if (CountingFacetsCollector.assertParams(fsp) == null) {
return new CountingFacetsCollector(fsp, taxoReader);
}
if (StandardFacetsCollector.assertParams(fsp) == null) {
return new StandardFacetsCollector(fsp, indexReader, taxoReader);
}
throw new IllegalArgumentException("None of the built-in FacetsCollectors can handle the given search params");
}
/**
* Returns a {@link FacetResult} per {@link FacetRequest} set in
* {@link FacetSearchParams}. Note that if one of the {@link FacetRequest

12
lucene/facet/src/java/org/apache/lucene/facet/search/StandardFacetsCollector.java
View File

@ -49,6 +49,17 @@ public class StandardFacetsCollector extends FacetsCollector {
private List<FacetResult> results;
private Object resultsGuard;
static String assertParams(FacetSearchParams fsp) {
// make sure none of the categories in the given FacetRequests was indexed with NO_PARENTS
for (FacetRequest fr : fsp.facetRequests) {
CategoryListParams clp = fsp.indexingParams.getCategoryListParams(fr.categoryPath);
if (clp.getOrdinalPolicy(fr.categoryPath.components[0]) == OrdinalPolicy.NO_PARENTS) {
return "this collector does not support aggregating categories that were indexed with OrdinalPolicy.NO_PARENTS";
}
}
return null;
}
/**
* Create a collector for accumulating facets while collecting documents
* during search.
@ -62,6 +73,7 @@ public class StandardFacetsCollector extends FacetsCollector {
* taxonomy containing the facets.
*/
public StandardFacetsCollector(FacetSearchParams facetSearchParams, IndexReader indexReader, TaxonomyReader taxonomyReader) {
assert assertParams(facetSearchParams) == null : assertParams(facetSearchParams);
facetsAccumulator = initFacetsAccumulator(facetSearchParams, indexReader, taxonomyReader);
scoreDocIdCollector = initScoredDocCollector(facetSearchParams, indexReader, taxonomyReader);
resultsGuard = new Object();

1
lucene/facet/src/java/org/apache/lucene/facet/util/ScoredDocIdsUtils.java
View File

@ -74,6 +74,7 @@ public class ScoredDocIdsUtils {
/** Clear all deleted documents from a given open-bit-set according to a given reader */
private static void clearDeleted(final IndexReader reader, final FixedBitSet set) throws IOException {
// TODO use BitsFilteredDocIdSet?
// If there are no deleted docs
if (!reader.hasDeletions()) {

8
lucene/facet/src/examples/org/apache/lucene/facet/example/merge/TaxonomyMergeUtils.java → lucene/facet/src/java/org/apache/lucene/facet/util/TaxonomyMergeUtils.java
View File

@ -1,9 +1,8 @@
package org.apache.lucene.facet.example.merge;
package org.apache.lucene.facet.util;
import java.io.IOException;
import java.util.List;
import org.apache.lucene.facet.example.ExampleUtils;
import org.apache.lucene.facet.index.OrdinalMappingAtomicReader;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
@ -17,6 +16,7 @@ import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Version;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@ -36,6 +36,7 @@ import org.apache.lucene.store.Directory;
*/
/**
* Utility methods for merging index and taxonomy directories.
* @lucene.experimental
*/
public class TaxonomyMergeUtils {
@ -52,8 +53,7 @@ public class TaxonomyMergeUtils {
*/
public static void merge(Directory srcIndexDir, Directory srcTaxDir, Directory destIndexDir, Directory destTaxDir,
FacetIndexingParams params) throws IOException {
IndexWriter destIndexWriter = new IndexWriter(destIndexDir,
new IndexWriterConfig(ExampleUtils.EXAMPLE_VER, null));
IndexWriter destIndexWriter = new IndexWriter(destIndexDir, new IndexWriterConfig(Version.LUCENE_42, null));
DirectoryTaxonomyWriter destTaxWriter = new DirectoryTaxonomyWriter(destTaxDir);
merge(srcIndexDir, srcTaxDir, new MemoryOrdinalMap(), destIndexWriter, destTaxWriter, params);
destTaxWriter.close();

61
lucene/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
View File

@ -6,6 +6,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@ -17,6 +18,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@ -44,6 +46,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase.SuppressCodecs;
import org.apache.lucene.util._TestUtil;
import org.apache.lucene.util.collections.IntToObjectMap;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@ -73,7 +76,8 @@ public abstract class FacetTestBase extends FacetTestCase {
SearchTaxoDirPair() {}
}
private static HashMap<Integer, SearchTaxoDirPair> dirsPerPartitionSize;
private static IntToObjectMap<SearchTaxoDirPair> dirsPerPartitionSize;
private static IntToObjectMap<FacetIndexingParams> fipPerPartitionSize;
private static File TEST_DIR;
/** Documents text field. */
@ -91,12 +95,15 @@ public abstract class FacetTestBase extends FacetTestCase {
@BeforeClass
public static void beforeClassFacetTestBase() {
TEST_DIR = _TestUtil.getTempDir("facets");
dirsPerPartitionSize = new HashMap<Integer, FacetTestBase.SearchTaxoDirPair>();
dirsPerPartitionSize = new IntToObjectMap<FacetTestBase.SearchTaxoDirPair>();
fipPerPartitionSize = new IntToObjectMap<FacetIndexingParams>();
}
@AfterClass
public static void afterClassFacetTestBase() throws Exception {
for (SearchTaxoDirPair pair : dirsPerPartitionSize.values()) {
Iterator<SearchTaxoDirPair> iter = dirsPerPartitionSize.iterator();
while (iter.hasNext()) {
SearchTaxoDirPair pair = iter.next();
IOUtils.close(pair.searchDir, pair.taxoDir);
}
}
@ -128,18 +135,14 @@ public abstract class FacetTestBase extends FacetTestCase {
return DEFAULT_CONTENT[doc];
}
/** Prepare index (in RAM) with single partition */
protected final void initIndex() throws Exception {
initIndex(Integer.MAX_VALUE);
/** Prepare index (in RAM) with some documents and some facets. */
protected final void initIndex(FacetIndexingParams fip) throws Exception {
initIndex(false, fip);
}
/** Prepare index (in RAM) with some documents and some facets */
protected final void initIndex(int partitionSize) throws Exception {
initIndex(partitionSize, false);
}
/** Prepare index (in RAM/Disk) with some documents and some facets */
protected final void initIndex(int partitionSize, boolean forceDisk) throws Exception {
/** Prepare index (in RAM/Disk) with some documents and some facets. */
protected final void initIndex(boolean forceDisk, FacetIndexingParams fip) throws Exception {
int partitionSize = fip.getPartitionSize();
if (VERBOSE) {
System.out.println("Partition Size: " + partitionSize + " forceDisk: "+forceDisk);
}
@ -158,7 +161,7 @@ public abstract class FacetTestBase extends FacetTestCase {
RandomIndexWriter iw = new RandomIndexWriter(random(), pair.searchDir, getIndexWriterConfig(getAnalyzer()));
TaxonomyWriter taxo = new DirectoryTaxonomyWriter(pair.taxoDir, OpenMode.CREATE);
populateIndex(iw, taxo, getFacetIndexingParams(partitionSize));
populateIndex(iw, taxo, fip);
// commit changes (taxonomy prior to search index for consistency)
taxo.commit();
@ -182,14 +185,40 @@ public abstract class FacetTestBase extends FacetTestCase {
/** Returns a {@link FacetIndexingParams} per the given partition size. */
protected FacetIndexingParams getFacetIndexingParams(final int partSize) {
return getFacetIndexingParams(partSize, false);
}
/**
* Returns a {@link FacetIndexingParams} per the given partition size. If
* requested, then {@link OrdinalPolicy} will be set to
* {@link OrdinalPolicy#ALL_PARENTS}, otherwise it will randomize.
*/
protected FacetIndexingParams getFacetIndexingParams(final int partSize, final boolean forceAllParents) {
FacetIndexingParams fip = fipPerPartitionSize.get(partSize);
if (fip == null) {
// randomize OrdinalPolicy. Since not all Collectors / Accumulators
// support NO_PARENTS, don't include it.
// TODO: once all code paths support NO_PARENTS, randomize it too.
CategoryListParams randomOP = new CategoryListParams() {
final OrdinalPolicy op = random().nextBoolean() ? OrdinalPolicy.ALL_BUT_DIMENSION : OrdinalPolicy.ALL_PARENTS;
@Override
public OrdinalPolicy getOrdinalPolicy(String dimension) {
return forceAllParents ? OrdinalPolicy.ALL_PARENTS : op;
}
};
// several of our encoders don't support the value 0,
// which is one of the values encoded when dealing w/ partitions.
return new FacetIndexingParams() {
// which is one of the values encoded when dealing w/ partitions,
// therefore don't randomize the encoder.
fip = new FacetIndexingParams(randomOP) {
@Override
public int getPartitionSize() {
return partSize;
}
};
fipPerPartitionSize.put(partSize, fip);
}
return fip;
}
/**

74
lucene/facet/src/test/org/apache/lucene/facet/FacetTestUtils.java
View File

@ -1,19 +1,7 @@
package org.apache.lucene.facet;
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@ -34,68 +22,6 @@ import org.apache.lucene.util.LuceneTestCase;
public class FacetTestUtils {
public static class IndexTaxonomyReaderPair {
public DirectoryReader indexReader;
public DirectoryTaxonomyReader taxReader;
public IndexSearcher indexSearcher;
public void close() throws IOException {
indexReader.close();
taxReader.close();
}
}
public static class IndexTaxonomyWriterPair {
public IndexWriter indexWriter;
public TaxonomyWriter taxWriter;
public void close() throws IOException {
indexWriter.close();
taxWriter.close();
}
public void commit() throws IOException {
indexWriter.commit();
taxWriter.commit();
}
}
public static Directory[][] createIndexTaxonomyDirs(int number) {
Directory[][] dirs = new Directory[number][2];
for (int i = 0; i < number; i++) {
dirs[i][0] = LuceneTestCase.newDirectory();
dirs[i][1] = LuceneTestCase.newDirectory();
}
return dirs;
}
public static IndexTaxonomyReaderPair[] createIndexTaxonomyReaderPair(Directory[][] dirs) throws IOException {
IndexTaxonomyReaderPair[] pairs = new IndexTaxonomyReaderPair[dirs.length];
for (int i = 0; i < dirs.length; i++) {
IndexTaxonomyReaderPair pair = new IndexTaxonomyReaderPair();
pair.indexReader = DirectoryReader.open(dirs[i][0]);
pair.indexSearcher = new IndexSearcher(pair.indexReader);
pair.taxReader = new DirectoryTaxonomyReader(dirs[i][1]);
pairs[i] = pair;
}
return pairs;
}
public static IndexTaxonomyWriterPair[] createIndexTaxonomyWriterPair(Directory[][] dirs) throws IOException {
IndexTaxonomyWriterPair[] pairs = new IndexTaxonomyWriterPair[dirs.length];
for (int i = 0; i < dirs.length; i++) {
IndexTaxonomyWriterPair pair = new IndexTaxonomyWriterPair();
pair.indexWriter = new IndexWriter(dirs[i][0], new IndexWriterConfig(
LuceneTestCase.TEST_VERSION_CURRENT, new MockAnalyzer(LuceneTestCase.random())));
pair.taxWriter = new DirectoryTaxonomyWriter(dirs[i][1]);
pair.indexWriter.commit();
pair.taxWriter.commit();
pairs[i] = pair;
}
return pairs;
}
public static String toSimpleString(FacetResult fr) {
StringBuilder sb = new StringBuilder();
toSimpleString(0, sb, fr.getFacetResultNode(), "");

2
lucene/facet/src/test/org/apache/lucene/facet/index/OrdinalMappingReaderTest.java
View File

@ -8,7 +8,6 @@ import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.example.merge.TaxonomyMergeUtils;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
@ -18,6 +17,7 @@ import org.apache.lucene.facet.search.results.FacetResultNode;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.facet.util.TaxonomyMergeUtils;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;

22
lucene/facet/src/test/org/apache/lucene/facet/index/TestFacetsPayloadMigrationReader.java
View File

@ -25,6 +25,8 @@ import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
import org.apache.lucene.facet.search.CategoryListIterator;
import org.apache.lucene.facet.search.DrillDown;
import org.apache.lucene.facet.search.FacetsCollector;
@ -368,9 +370,23 @@ public class TestFacetsPayloadMigrationReader extends FacetTestCase {
// set custom CLP fields for two dimensions and use the default ($facets) for the other two
HashMap<CategoryPath,CategoryListParams> params = new HashMap<CategoryPath,CategoryListParams>();
params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]));
params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]));
FacetIndexingParams fip = new PerDimensionIndexingParams(params) {
params.put(new CategoryPath(DIMENSIONS[0]), new CategoryListParams(DIMENSIONS[0]) {
@Override
public OrdinalPolicy getOrdinalPolicy(String dimension) {
return OrdinalPolicy.ALL_PARENTS;
}
});
params.put(new CategoryPath(DIMENSIONS[1]), new CategoryListParams(DIMENSIONS[1]) {
@Override
public OrdinalPolicy getOrdinalPolicy(String dimension) {
return OrdinalPolicy.ALL_PARENTS;
}
});
HashMap<String,OrdinalPolicy> policies = new HashMap<String,CategoryListParams.OrdinalPolicy>();
policies.put(DIMENSIONS[2], OrdinalPolicy.ALL_PARENTS);
policies.put(DIMENSIONS[3], OrdinalPolicy.ALL_PARENTS);
FacetIndexingParams fip = new PerDimensionIndexingParams(params, new PerDimensionOrdinalPolicy(policies)) {
@Override
public int getPartitionSize() {
return partitionSize;

13
lucene/facet/src/test/org/apache/lucene/facet/search/BaseTestTopK.java
View File

@ -51,10 +51,9 @@ public abstract class BaseTestTopK extends FacetTestBase {
private int nextInt;
@Override
protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo,
FacetIndexingParams iParams) throws IOException {
protected void populateIndex(RandomIndexWriter iw, TaxonomyWriter taxo, FacetIndexingParams fip) throws IOException {
currDoc = -1;
super.populateIndex(iw, taxo, iParams);
super.populateIndex(iw, taxo, fip);
}
/** prepare the next random int */
@ -94,17 +93,13 @@ public abstract class BaseTestTopK extends FacetTestBase {
return Arrays.asList(cp);
}
protected FacetSearchParams searchParamsWithRequests(int numResults) {
return searchParamsWithRequests(numResults, Integer.MAX_VALUE);
}
protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
facetRequests.add(new CountFacetRequest(new CategoryPath("a"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "1", "10"), numResults));
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "2", "26", "267"), numResults));
return getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
return getFacetSearchParams(facetRequests, fip);
}
@Override

27
lucene/facet/src/test/org/apache/lucene/facet/search/CountingFacetsCollectorTest.java
View File

@ -16,8 +16,9 @@ import org.apache.lucene.document.StringField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.index.params.PerDimensionIndexingParams;
import org.apache.lucene.facet.index.params.PerDimensionOrdinalPolicy;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest.SortBy;
@ -146,13 +147,11 @@ public class CountingFacetsCollectorTest extends FacetTestCase {
termExpectedCounts.put(cp, termExpectedCounts.get(cp) + 1);
}
}
// add 1 to each dimension
allExpectedCounts.put(CP_A, allExpectedCounts.get(CP_A) + 1);
// add 1 to each NO_PARENTS dimension
allExpectedCounts.put(CP_B, allExpectedCounts.get(CP_B) + 1);
allExpectedCounts.put(CP_C, allExpectedCounts.get(CP_C) + 1);
allExpectedCounts.put(CP_D, allExpectedCounts.get(CP_D) + 1);
if (updateTermExpectedCounts) {
termExpectedCounts.put(CP_A, termExpectedCounts.get(CP_A) + 1);
termExpectedCounts.put(CP_B, termExpectedCounts.get(CP_B) + 1);
termExpectedCounts.put(CP_C, termExpectedCounts.get(CP_C) + 1);
termExpectedCounts.put(CP_D, termExpectedCounts.get(CP_D) + 1);
@ -252,19 +251,13 @@ public class CountingFacetsCollectorTest extends FacetTestCase {
conf.setMergePolicy(NoMergePolicy.COMPOUND_FILES); // prevent merges, so we can control the index segments
IndexWriter indexWriter = new IndexWriter(indexDir, conf);
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
CategoryListParams allParents = new CategoryListParams();
CategoryListParams noParents = new CategoryListParams("no_parents") {
@Override
public OrdinalPolicy getOrdinalPolicy() {
return OrdinalPolicy.NO_PARENTS;
}
};
Map<CategoryPath,CategoryListParams> params = new HashMap<CategoryPath,CategoryListParams>();
params.put(CP_A, allParents);
params.put(CP_B, allParents);
params.put(CP_C, noParents);
params.put(CP_D, noParents);
fip = new PerDimensionIndexingParams(params);
Map<String,OrdinalPolicy> policies = new HashMap<String,CategoryListParams.OrdinalPolicy>();
policies.put(CP_B.components[0], OrdinalPolicy.ALL_PARENTS);
policies.put(CP_C.components[0], OrdinalPolicy.NO_PARENTS);
policies.put(CP_D.components[0], OrdinalPolicy.NO_PARENTS);
CategoryListParams clp = new PerDimensionOrdinalPolicy(policies);
fip = new FacetIndexingParams(clp);
allExpectedCounts = newCounts();
termExpectedCounts = newCounts();

6
lucene/facet/src/test/org/apache/lucene/facet/search/TestDemoFacets.java
View File

@ -104,9 +104,9 @@ public class TestDemoFacets extends FacetTestCase {
// Retrieve & verify results:
List<FacetResult> results = c.getFacetResults();
assertEquals(2, results.size());
assertEquals("Publish Date (5)\n 2012 (2)\n 2010 (2)\n 1999 (1)\n",
assertEquals("Publish Date (0)\n 2012 (2)\n 2010 (2)\n 1999 (1)\n",
FacetTestUtils.toSimpleString(results.get(0)));
assertEquals("Author (5)\n Lisa (2)\n Frank (1)\n Susan (1)\n Bob (1)\n",
assertEquals("Author (0)\n Lisa (2)\n Frank (1)\n Susan (1)\n Bob (1)\n",
FacetTestUtils.toSimpleString(results.get(1)));
@ -117,7 +117,7 @@ public class TestDemoFacets extends FacetTestCase {
searcher.search(q2, c);
results = c.getFacetResults();
assertEquals(1, results.size());
assertEquals("Author (2)\n Lisa (1)\n Bob (1)\n",
assertEquals("Author (0)\n Lisa (1)\n Bob (1)\n",
FacetTestUtils.toSimpleString(results.get(0)));
// Smoke test PrintTaxonomyStats:

8
lucene/facet/src/test/org/apache/lucene/facet/search/TestFacetsAccumulatorWithComplement.java
View File

@ -14,6 +14,7 @@ import org.junit.Before;
import org.junit.Test;
import org.apache.lucene.facet.FacetTestBase;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.ScoredDocIDs;
import org.apache.lucene.facet.search.ScoredDocIdCollector;
@ -48,11 +49,14 @@ import org.apache.lucene.facet.taxonomy.CategoryPath;
*/
public class TestFacetsAccumulatorWithComplement extends FacetTestBase {
private FacetIndexingParams fip;
@Override
@Before
public void setUp() throws Exception {
super.setUp();
initIndex();
fip = getFacetIndexingParams(Integer.MAX_VALUE);
initIndex(fip);
}
@Override
@ -125,7 +129,7 @@ public class TestFacetsAccumulatorWithComplement extends FacetTestBase {
/** compute facets with certain facet requests and docs */
private List<FacetResult> findFacets(ScoredDocIDs sDocids, boolean withComplement) throws IOException {
FacetSearchParams fsp = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(new CategoryPath("root","a"), 10));
FacetSearchParams fsp = new FacetSearchParams(fip, new CountFacetRequest(new CategoryPath("root","a"), 10));
FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(fsp, indexReader, taxoReader);
fAccumulator.setComplementThreshold(

69
lucene/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
View File

@ -14,7 +14,6 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.FacetTestUtils;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
@ -77,12 +76,14 @@ public class TestMultipleCategoryLists extends FacetTestCase {
@Test
public void testDefault() throws Exception {
Directory[][] dirs = getDirs();
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
// create and open an index writer
RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
PerDimensionIndexingParams iParams = new PerDimensionIndexingParams(Collections.<CategoryPath, CategoryListParams>emptyMap());
@ -92,7 +93,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@ -105,17 +106,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$facets", ir);
IOUtils.close(tr, ir, iw, tw);
IOUtils.close(dirs[0]);
IOUtils.close(indexDir, taxoDir);
}
@Test
public void testCustom() throws Exception {
Directory[][] dirs = getDirs();
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
// create and open an index writer
RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
PerDimensionIndexingParams iParams = new PerDimensionIndexingParams(
Collections.singletonMap(new CategoryPath("Author"), new CategoryListParams("$author")));
@ -125,7 +128,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@ -139,17 +142,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$author", ir);
IOUtils.close(tr, ir, iw, tw);
IOUtils.close(dirs[0]);
IOUtils.close(indexDir, taxoDir);
}
@Test
public void testTwoCustomsSameField() throws Exception {
Directory[][] dirs = getDirs();
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
// create and open an index writer
RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$music"));
@ -161,7 +166,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@ -176,7 +181,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$music", ir);
IOUtils.close(tr, ir, iw, tw);
IOUtils.close(dirs[0]);
IOUtils.close(indexDir, taxoDir);
}
private void assertOrdinalsExist(String field, IndexReader ir) throws IOException {
@ -191,12 +196,14 @@ public class TestMultipleCategoryLists extends FacetTestCase {
@Test
public void testDifferentFieldsAndText() throws Exception {
Directory[][] dirs = getDirs();
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
// create and open an index writer
RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$bands"));
@ -208,7 +215,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@ -222,17 +229,19 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$composers", ir);
IOUtils.close(tr, ir, iw, tw);
IOUtils.close(dirs[0]);
IOUtils.close(indexDir, taxoDir);
}
@Test
public void testSomeSameSomeDifferent() throws Exception {
Directory[][] dirs = getDirs();
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
// create and open an index writer
RandomIndexWriter iw = new RandomIndexWriter(random(), dirs[0][0], newIndexWriterConfig(
RandomIndexWriter iw = new RandomIndexWriter(random(), indexDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
// create and open a taxonomy writer
TaxonomyWriter tw = new DirectoryTaxonomyWriter(dirs[0][1], OpenMode.CREATE);
TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir, OpenMode.CREATE);
Map<CategoryPath,CategoryListParams> paramsMap = new HashMap<CategoryPath,CategoryListParams>();
paramsMap.put(new CategoryPath("Band"), new CategoryListParams("$music"));
@ -246,7 +255,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
tw.commit();
// prepare index reader and taxonomy.
TaxonomyReader tr = new DirectoryTaxonomyReader(dirs[0][1]);
TaxonomyReader tr = new DirectoryTaxonomyReader(taxoDir);
// prepare searcher to search against
IndexSearcher searcher = newSearcher(ir);
@ -259,11 +268,7 @@ public class TestMultipleCategoryLists extends FacetTestCase {
assertOrdinalsExist("$literature", ir);
IOUtils.close(tr, ir, iw, tw);
IOUtils.close(dirs[0]);
}
private Directory[][] getDirs() {
return FacetTestUtils.createIndexTaxonomyDirs(1);
IOUtils.close(indexDir, taxoDir);
}
private void assertCorrectResults(FacetsCollector facetsCollector) throws IOException {
@ -274,7 +279,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
Iterable<? extends FacetResultNode> subResults = resNode.subResults;
Iterator<? extends FacetResultNode> subIter = subResults.iterator();
checkResult(resNode, "Band", 5.0);
checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Punk", 1.0);
@ -283,7 +287,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
subResults = resNode.subResults;
subIter = subResults.iterator();
checkResult(resNode, "Band", 5.0);
checkResult(subIter.next(), "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
@ -297,7 +300,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
subResults = resNode.subResults;
subIter = subResults.iterator();
checkResult(resNode, "Author", 3.0);
checkResult(subIter.next(), "Author/Kurt Vonnegut", 1.0);
checkResult(subIter.next(), "Author/Stephen King", 1.0);
checkResult(subIter.next(), "Author/Mark Twain", 1.0);
@ -307,7 +309,6 @@ public class TestMultipleCategoryLists extends FacetTestCase {
subResults = resNode.subResults;
subIter = subResults.iterator();
checkResult(resNode, "Band/Rock & Pop", 4.0);
checkResult(subIter.next(), "Band/Rock & Pop/Dave Matthews Band", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/REM", 1.0);
checkResult(subIter.next(), "Band/Rock & Pop/U2", 1.0);

10
lucene/facet/src/test/org/apache/lucene/facet/search/TestSameRequestAccumulation.java
View File

@ -3,6 +3,7 @@ package org.apache.lucene.facet.search;
import java.util.List;
import org.apache.lucene.facet.FacetTestBase;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.FacetsCollector;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
@ -31,18 +32,21 @@ import org.junit.Before;
public class TestSameRequestAccumulation extends FacetTestBase {
private FacetIndexingParams fip;
@Override
@Before
public void setUp() throws Exception {
super.setUp();
initIndex();
fip = getFacetIndexingParams(Integer.MAX_VALUE);
initIndex(fip);
}
// Following LUCENE-4461 - ensure requesting the (exact) same request more
// than once does not alter the results
public void testTwoSameRequests() throws Exception {
final CountFacetRequest facetRequest = new CountFacetRequest(new CategoryPath("root"), 10);
FacetSearchParams fsp = new FacetSearchParams(facetRequest);
FacetSearchParams fsp = new FacetSearchParams(fip, facetRequest);
FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
searcher.search(new MatchAllDocsQuery(), fc);
@ -50,7 +54,7 @@ public class TestSameRequestAccumulation extends FacetTestBase {
final String expected = fc.getFacetResults().get(0).toString();
// now add the same facet request with duplicates (same instance and same one)
fsp = new FacetSearchParams(facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
fsp = new FacetSearchParams(fip, facetRequest, facetRequest, new CountFacetRequest(new CategoryPath("root"), 10));
// make sure the search params holds 3 requests now
assertEquals(3, fsp.facetRequests.size());

16
lucene/facet/src/test/org/apache/lucene/facet/search/TestScoredDocIdCollector.java
View File

@ -5,6 +5,7 @@ import java.util.Arrays;
import java.util.List;
import org.apache.lucene.facet.FacetTestBase;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.params.ScoreFacetRequest;
@ -37,11 +38,14 @@ import org.junit.Test;
/** Test ScoredDocIdCollector. */
public class TestScoredDocIdCollector extends FacetTestBase {
private FacetIndexingParams fip;
@Override
@Before
public void setUp() throws Exception {
super.setUp();
initIndex();
fip = getFacetIndexingParams(Integer.MAX_VALUE);
initIndex(fip);
}
@Override
@ -73,8 +77,8 @@ public class TestScoredDocIdCollector extends FacetTestBase {
// verify by facet values
CategoryPath cp = new CategoryPath("root","a");
FacetSearchParams countFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new CountFacetRequest(cp, 10));
FacetSearchParams scoreFSP = new FacetSearchParams(getFacetIndexingParams(Integer.MAX_VALUE), new ScoreFacetRequest(cp, 10));
FacetSearchParams countFSP = new FacetSearchParams(fip, new CountFacetRequest(cp, 10));
FacetSearchParams scoreFSP = new FacetSearchParams(fip, new ScoreFacetRequest(cp, 10));
List<FacetResult> countRes = findFacets(scoredDocIDs, countFSP);
List<FacetResult> scoreRes = findFacets(scoredDocIDs, scoreFSP);
@ -101,10 +105,8 @@ public class TestScoredDocIdCollector extends FacetTestBase {
}
// compute facets with certain facet requests and docs
private List<FacetResult> findFacets(ScoredDocIDs sDocids,
FacetSearchParams facetSearchParams) throws IOException {
FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(
facetSearchParams, indexReader, taxoReader);
private List<FacetResult> findFacets(ScoredDocIDs sDocids, FacetSearchParams facetSearchParams) throws IOException {
FacetsAccumulator fAccumulator = new StandardFacetsAccumulator(facetSearchParams, indexReader, taxoReader);
List<FacetResult> res = fAccumulator.accumulate(sDocids);
// Results are ready, printing them...

1
lucene/facet/src/test/org/apache/lucene/facet/search/TestStandardFacetsAccumulator.java
View File

@ -113,7 +113,6 @@ public class TestStandardFacetsAccumulator extends FacetTestCase {
List<FacetResult> results = fc.getFacetResults();
assertEquals("received too many facet results", 1, results.size());
FacetResultNode frn = results.get(0).getFacetResultNode();
assertEquals("wrong weight for \"A\"", 4, (int) frn.value);
assertEquals("wrong number of children", 2, frn.subResults.size());
for (FacetResultNode node : frn.subResults) {
assertEquals("wrong weight for child " + node.label, 2, (int) node.value);

3
lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
View File

@ -181,7 +181,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
boolean hasDoctor = "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(9, fr.getNumValidDescendants());
FacetResultNode parentRes = fr.getFacetResultNode();
assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values: a/b with 8 and a/c with 6
// a/b has two children a/b/2 with value 3, and a/b/1 with value 2.
@ -217,7 +216,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(9, fr.getNumValidDescendants());
parentRes = fr.getFacetResultNode();
assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values: a/b with 8 and a/c with 6
// a/b has two children a/b/2 with value 3, and a/b/1 with value 2.
@ -234,7 +232,6 @@ public class TestTopKInEachNodeResultHandler extends FacetTestCase {
hasDoctor |= "Doctor".equals(fr.getFacetRequest().categoryPath.components[0]);
assertEquals(4, fr.getNumValidDescendants(), 4);
parentRes = fr.getFacetResultNode();
assertEquals(16.0, parentRes.value, Double.MIN_VALUE);
assertEquals(2, parentRes.subResults.size());
// two nodes sorted by descending values:
// a/b with value 8 and a/c with value 6

33
lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandler.java
View File

@ -4,6 +4,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import org.apache.lucene.facet.index.params.CategoryListParams.OrdinalPolicy;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest;
import org.apache.lucene.facet.search.params.FacetRequest.ResultMode;
@ -73,7 +75,9 @@ public class TestTopKResultsHandler extends BaseTestTopK {
@Test
public void testSimple() throws Exception {
for (int partitionSize : partitionSizes) {
initIndex(partitionSize);
FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
OrdinalPolicy op = fip.getCategoryListParams(null).getOrdinalPolicy(null);
initIndex(fip);
List<FacetRequest> facetRequests = new ArrayList<FacetRequest>();
facetRequests.add(new CountFacetRequest(new CategoryPath("a"), 100));
@ -87,7 +91,7 @@ public class TestTopKResultsHandler extends BaseTestTopK {
facetRequests.add(new CountFacetRequest(new CategoryPath("a", "c"), 100));
// do different facet counts and compare to control
FacetSearchParams sParams = getFacetSearchParams(facetRequests, getFacetIndexingParams(partitionSize));
FacetSearchParams sParams = getFacetSearchParams(facetRequests, fip);
FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
@Override
@ -103,14 +107,18 @@ public class TestTopKResultsHandler extends BaseTestTopK {
FacetResult fr = facetResults.get(0);
FacetResultNode parentRes = fr.getFacetResultNode();
if (op == OrdinalPolicy.ALL_PARENTS) {
assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
}
FacetResultNode[] frn = resultNodesAsArray(parentRes);
assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
fr = facetResults.get(1);
parentRes = fr.getFacetResultNode();
if (op == OrdinalPolicy.ALL_PARENTS) {
assertEquals(13.0, parentRes.value, Double.MIN_VALUE);
}
frn = resultNodesAsArray(parentRes);
assertEquals(7.0, frn[0].value, Double.MIN_VALUE);
assertEquals(6.0, frn[1].value, Double.MIN_VALUE);
@ -121,7 +129,9 @@ public class TestTopKResultsHandler extends BaseTestTopK {
fr = facetResults.get(2);
parentRes = fr.getFacetResultNode();
if (op == OrdinalPolicy.ALL_PARENTS) {
assertEquals(7.0, parentRes.value, Double.MIN_VALUE);
}
frn = resultNodesAsArray(parentRes);
assertEquals(2.0, frn[0].value, Double.MIN_VALUE);
assertEquals(2.0, frn[1].value, Double.MIN_VALUE);
@ -130,13 +140,17 @@ public class TestTopKResultsHandler extends BaseTestTopK {
fr = facetResults.get(3);
parentRes = fr.getFacetResultNode();
if (op == OrdinalPolicy.ALL_PARENTS) {
assertEquals(2.0, parentRes.value, Double.MIN_VALUE);
}
frn = resultNodesAsArray(parentRes);
assertEquals(0, frn.length);
fr = facetResults.get(4);
parentRes = fr.getFacetResultNode();
if (op == OrdinalPolicy.ALL_PARENTS) {
assertEquals(6.0, parentRes.value, Double.MIN_VALUE);
}
frn = resultNodesAsArray(parentRes);
assertEquals(1.0, frn[0].value, Double.MIN_VALUE);
closeAll();
@ -149,12 +163,12 @@ public class TestTopKResultsHandler extends BaseTestTopK {
@Test
public void testGetMaxIntFacets() throws Exception {
for (int partitionSize : partitionSizes) {
initIndex(partitionSize);
FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
initIndex(fip);
// do different facet counts and compare to control
CategoryPath path = new CategoryPath("a", "b");
FacetSearchParams sParams = getFacetSearchParams(getFacetIndexingParams(partitionSize),
new CountFacetRequest(path, Integer.MAX_VALUE));
FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, Integer.MAX_VALUE));
FacetsCollector fc = new StandardFacetsCollector(sParams, indexReader, taxoReader) {
@Override
@ -174,7 +188,7 @@ public class TestTopKResultsHandler extends BaseTestTopK {
// As a control base results, ask for top-1000 results
FacetSearchParams sParams2 = getFacetSearchParams(
getFacetIndexingParams(partitionSize), new CountFacetRequest(path, Integer.MAX_VALUE));
fip, new CountFacetRequest(path, Integer.MAX_VALUE));
FacetsCollector fc2 = new StandardFacetsCollector(sParams2, indexReader, taxoReader) {
@Override
@ -207,12 +221,11 @@ public class TestTopKResultsHandler extends BaseTestTopK {
@Test
public void testSimpleSearchForNonexistentFacet() throws Exception {
for (int partitionSize : partitionSizes) {
initIndex(partitionSize);
FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
initIndex(fip);
CategoryPath path = new CategoryPath("Miau Hattulla");
FacetSearchParams sParams = getFacetSearchParams(
getFacetIndexingParams(partitionSize),
new CountFacetRequest(path, 10));
FacetSearchParams sParams = getFacetSearchParams(fip, new CountFacetRequest(path, 10));
FacetsCollector fc = FacetsCollector.create(sParams, indexReader, taxoReader);

16
lucene/facet/src/test/org/apache/lucene/facet/search/TestTopKResultsHandlerRandom.java
View File

@ -4,6 +4,7 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
@ -32,10 +33,10 @@ import org.junit.Test;
public class TestTopKResultsHandlerRandom extends BaseTestTopK {
private List<FacetResult> countFacets(int partitionSize, int numResults, final boolean doComplement)
private List<FacetResult> countFacets(FacetIndexingParams fip, int numResults, final boolean doComplement)
throws IOException {
Query q = new MatchAllDocsQuery();
FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, partitionSize);
FacetSearchParams facetSearchParams = searchParamsWithRequests(numResults, fip);
FacetsCollector fc = new StandardFacetsCollector(facetSearchParams, indexReader, taxoReader) {
@Override
protected FacetsAccumulator initFacetsAccumulator(
@ -59,7 +60,8 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
@Test
public void testTopCountsOrder() throws Exception {
for (int partitionSize : partitionSizes) {
initIndex(partitionSize);
FacetIndexingParams fip = getFacetIndexingParams(partitionSize);
initIndex(fip);
/*
* Try out faceted search in it's most basic form (no sampling nor complement
@ -67,7 +69,7 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
* being indexed, and later on an "over-all" faceted search is performed. The
* results are checked against the DF of each facet by itself
*/
List<FacetResult> facetResults = countFacets(partitionSize, 100000, false);
List<FacetResult> facetResults = countFacets(fip, 100000, false);
assertCountsAndCardinality(facetCountsTruth(), facetResults);
/*
@ -77,10 +79,10 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
* place in here. The results are checked against the a regular (a.k.a
* no-complement, no-sampling) faceted search with the same parameters.
*/
facetResults = countFacets(partitionSize, 100000, true);
facetResults = countFacets(fip, 100000, true);
assertCountsAndCardinality(facetCountsTruth(), facetResults);
List<FacetResult> allFacetResults = countFacets(partitionSize, 100000, false);
List<FacetResult> allFacetResults = countFacets(fip, 100000, false);
HashMap<String,Integer> all = new HashMap<String,Integer>();
int maxNumNodes = 0;
@ -108,7 +110,7 @@ public class TestTopKResultsHandlerRandom extends BaseTestTopK {
if (VERBOSE) {
System.out.println("------- verify for "+n+" top results");
}
List<FacetResult> someResults = countFacets(partitionSize, n, false);
List<FacetResult> someResults = countFacets(fip, n, false);
k = 0;
for (FacetResult fr : someResults) {
FacetResultNode topResNode = fr.getFacetResultNode();

71
lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCounts.java
View File

@ -3,12 +3,20 @@ package org.apache.lucene.facet.search;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.FacetTestUtils;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.CategoryListParams;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyReader;
import org.apache.lucene.facet.taxonomy.directory.DirectoryTaxonomyWriter;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util._TestUtil;
@ -33,9 +41,9 @@ import org.junit.Test;
public class TestTotalFacetCounts extends FacetTestCase {
private static void initCache(int numEntries) {
private static void initCache() {
TotalFacetCountsCache.getSingleton().clear();
TotalFacetCountsCache.getSingleton().setCacheSize(numEntries); // Set to keep one in mem
TotalFacetCountsCache.getSingleton().setCacheSize(1); // Set to keep one in mem
}
@Test
@ -48,48 +56,57 @@ public class TestTotalFacetCounts extends FacetTestCase {
}
private void doTestWriteRead(final int partitionSize) throws IOException {
initCache(1);
initCache();
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
// Create temporary RAMDirectories
Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
// Create our index/taxonomy writers
IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
FacetIndexingParams iParams = new FacetIndexingParams() {
@Override
public int getPartitionSize() {
return partitionSize;
}
@Override
public CategoryListParams getCategoryListParams(CategoryPath category) {
return new CategoryListParams() {
@Override
public OrdinalPolicy getOrdinalPolicy(String dimension) {
return OrdinalPolicy.ALL_PARENTS;
}
};
}
};
// The counts that the TotalFacetCountsArray should have after adding
// the below facets to the index.
int[] expectedCounts = new int[] { 0, 3, 1, 3, 1, 1, 1, 1, 1, 2, 1, 1, 1, 1 };
String[] categories = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };
// Add a facet to the index
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "e");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "d");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "g");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "z");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "a");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "1", "2");
TestTotalFacetCountsCache.addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "b", "c");
FacetFields facetFields = new FacetFields(taxoWriter, iParams);
for (String cat : categories) {
Document doc = new Document();
facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
indexWriter.addDocument(doc);
}
// Commit Changes
writers[0].close();
IOUtils.close(indexWriter, taxoWriter);
IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
int[] intArray = new int[iParams.getPartitionSize()];
TotalFacetCountsCache tfcc = TotalFacetCountsCache.getSingleton();
File tmpFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
tfcc.store(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
tfcc.store(tmpFile, indexReader, taxoReader, iParams);
tfcc.clear(); // not really required because TFCC overrides on load(), but in the test we need not rely on this.
tfcc.load(tmpFile, readers[0].indexReader, readers[0].taxReader, iParams);
tfcc.load(tmpFile, indexReader, taxoReader, iParams);
// now retrieve the one just loaded
TotalFacetCounts totalCounts = tfcc.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
TotalFacetCounts totalCounts = tfcc.getTotalCounts(indexReader, taxoReader, iParams);
int partition = 0;
for (int i = 0; i < expectedCounts.length; i += partitionSize) {
@ -103,8 +120,8 @@ public class TestTotalFacetCounts extends FacetTestCase {
Arrays.equals(partitionExpectedCounts, intArray));
++partition;
}
readers[0].close();
IOUtils.close(dirs[0]);
IOUtils.close(indexReader, taxoReader);
IOUtils.close(indexDir, taxoDir);
tmpFile.delete();
}

294
lucene/facet/src/test/org/apache/lucene/facet/search/TestTotalFacetCountsCache.java
View File

@ -4,21 +4,17 @@ import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.FacetTestUtils;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyReaderPair;
import org.apache.lucene.facet.FacetTestUtils.IndexTaxonomyWriterPair;
import org.apache.lucene.facet.example.ExampleResult;
import org.apache.lucene.facet.example.TestMultiCLExample;
import org.apache.lucene.facet.example.multiCL.MultiCLIndexer;
import org.apache.lucene.facet.example.multiCL.MultiCLSearcher;
import org.apache.lucene.facet.index.FacetFields;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.TotalFacetCounts.CreationType;
import org.apache.lucene.facet.search.params.CountFacetRequest;
import org.apache.lucene.facet.search.params.FacetSearchParams;
import org.apache.lucene.facet.search.results.FacetResult;
import org.apache.lucene.facet.search.results.FacetResultNode;
import org.apache.lucene.facet.taxonomy.CategoryPath;
@ -30,6 +26,8 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.IOUtils;
@ -86,7 +84,7 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
}
/** Utility method to add a document and facets to an index/taxonomy. */
static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
private static void addFacets(FacetIndexingParams iParams, IndexWriter iw,
TaxonomyWriter tw, String... strings) throws IOException {
Document doc = new Document();
FacetFields facetFields = new FacetFields(tw, iParams);
@ -95,7 +93,7 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
}
/** Clears the cache and sets its size to one. */
static void initCache() {
private static void initCache() {
TFC.clear();
TFC.setCacheSize(1); // Set to keep one in memory
}
@ -107,37 +105,35 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
initCache();
}
/** runs a few instances of {@link MultiCLSearcher} in parallel */
/** runs few searches in parallel */
public void testGeneralSynchronization() throws Exception {
int numIters = atLeast(2);
int numIters = atLeast(4);
Random random = random();
for (int i = 0; i < numIters; i++) {
doTestGeneralSynchronization(_TestUtil.nextInt(random(), 2, 4),
random().nextBoolean() ? -1 : _TestUtil.nextInt(random(), 1, 10),
_TestUtil.nextInt(random(), 0, 3));
int numThreads = random.nextInt(3) + 2; // 2-4
int sleepMillis = random.nextBoolean() ? -1 : random.nextInt(10) + 1 /*1-10*/;
int cacheSize = random.nextInt(4); // 0-3
doTestGeneralSynchronization(numThreads, sleepMillis, cacheSize);
}
}
/**
* Run many instances of {@link MultiCLSearcher} in parallel, results should
* be sane. Each instance has a random delay for reading bytes, to ensure
* that threads finish in different order than started.
*/
@Test @Nightly
public void testGeneralSynchronizationBig() throws Exception {
int[] numThreads = new int[] { 2, 3, 5, 8 };
int[] sleepMillis = new int[] { -1, 1, 20, 33 };
int[] cacheSize = new int[] { 0,1,2,3,5 };
for (int size : cacheSize) {
for (int sleep : sleepMillis) {
for (int nThreads : numThreads) {
doTestGeneralSynchronization(nThreads, sleep, size);
}
}
}
private static final String[] CATEGORIES = new String[] { "a/b", "c/d", "a/e", "a/d", "c/g", "c/z", "b/a", "1/2", "b/c" };
private void index(Directory indexDir, Directory taxoDir) throws IOException {
IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
FacetFields facetFields = new FacetFields(taxoWriter);
for (String cat : CATEGORIES) {
Document doc = new Document();
facetFields.addFields(doc, Collections.singletonList(new CategoryPath(cat, '/')));
indexWriter.addDocument(doc);
}
private void doTestGeneralSynchronization(int numThreads, int sleepMillis,
int cacheSize) throws Exception {
IOUtils.close(indexWriter, taxoWriter);
}
private void doTestGeneralSynchronization(int numThreads, int sleepMillis, int cacheSize) throws Exception {
TFC.setCacheSize(cacheSize);
SlowRAMDirectory slowIndexDir = new SlowRAMDirectory(-1, random());
MockDirectoryWrapper indexDir = new MockDirectoryWrapper(random(), slowIndexDir);
@ -145,7 +141,7 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
MockDirectoryWrapper taxoDir = new MockDirectoryWrapper(random(), slowTaxoDir);
// Index documents without the "slowness"
MultiCLIndexer.index(indexDir, taxoDir);
index(indexDir, taxoDir);
slowIndexDir.setSleepMillis(sleepMillis);
slowTaxoDir.setSleepMillis(sleepMillis);
@ -161,80 +157,64 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
private IndexReader indexReader;
private TaxonomyReader taxoReader;
public Multi(IndexReader indexReader, TaxonomyReader taxoReader,
FacetIndexingParams iParams) {
public Multi(IndexReader indexReader, TaxonomyReader taxoReader, FacetIndexingParams iParams) {
this.indexReader = indexReader;
this.taxoReader = taxoReader;
this.iParams = iParams;
}
public ExampleResult getResults() {
ExampleResult exampleRes = new ExampleResult();
exampleRes.setFacetResults(results);
return exampleRes;
public List<FacetResult> getResults() {
return results;
}
@Override
public void run() {
try {
results = MultiCLSearcher.searchWithFacets(indexReader, taxoReader, iParams);
FacetSearchParams fsp = new FacetSearchParams(iParams, new CountFacetRequest(new CategoryPath("a"), 10),
new CountFacetRequest(new CategoryPath("b"), 10));
IndexSearcher searcher = new IndexSearcher(indexReader);
FacetsCollector fc = FacetsCollector.create(fsp, indexReader, taxoReader);
searcher.search(new MatchAllDocsQuery(), fc);
results = fc.getFacetResults();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
}
// Instantiate threads, but do not start them
Multi[] multis = new Multi[numThreads];
for (int i = 0; i < numThreads - 1; i++) {
multis[i] = new Multi(slowIndexReader, slowTaxoReader, MultiCLIndexer.MULTI_IPARAMS);
for (int i = 0; i < numThreads; i++) {
multis[i] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
}
// The last thread uses ONLY the DefaultFacetIndexingParams so that
// it references a different TFC cache. This will still result
// in valid results, but will only search one of the category lists
// instead of all of them.
multis[numThreads - 1] = new Multi(slowIndexReader, slowTaxoReader, FacetIndexingParams.ALL_PARENTS);
// Gentleman, start your engines
for (Multi m : multis) {
m.start();
}
// Wait for threads and get results
ExampleResult[] multiResults = new ExampleResult[numThreads];
for (int i = 0; i < numThreads; i++) {
multis[i].join();
multiResults[i] = multis[i].getResults();
String[] expLabelsA = new String[] { "a/d", "a/e", "a/b" };
String[] expLabelsB = new String[] { "b/c", "b/a" };
for (Multi m : multis) {
m.join();
List<FacetResult> facetResults = m.getResults();
assertEquals("expected two results", 2, facetResults.size());
FacetResultNode nodeA = facetResults.get(0).getFacetResultNode();
int i = 0;
for (FacetResultNode node : nodeA.subResults) {
assertEquals("wrong count", 1, (int) node.value);
assertEquals(expLabelsA[i++], node.label.toString('/'));
}
// Each of the (numThreads-1) should have the same predictable
// results, which we test for here.
for (int i = 0; i < numThreads - 1; i++) {
ExampleResult eResults = multiResults[i];
TestMultiCLExample.assertCorrectMultiResults(eResults);
FacetResultNode nodeB = facetResults.get(1).getFacetResultNode();
i = 0;
for (FacetResultNode node : nodeB.subResults) {
assertEquals("wrong count", 1, (int) node.value);
assertEquals(expLabelsB[i++], node.label.toString('/'));
}
}
// The last thread, which only searched over the
// DefaultFacetIndexingParams,
// has its own results
ExampleResult eResults = multiResults[numThreads - 1];
List<FacetResult> results = eResults.getFacetResults();
assertEquals(3, results.size());
String[] expLabels = new String[] { "5", "5/5", "6/2" };
double[] expValues = new double[] { 0.0, 0.0, 1.0 };
for (int i = 0; i < 3; i++) {
FacetResult result = results.get(i);
assertNotNull("Result should not be null", result);
FacetResultNode resNode = result.getFacetResultNode();
assertEquals("Invalid label", expLabels[i], resNode.label.toString());
assertEquals("Invalid value", expValues[i], resNode.value, 0.0);
assertEquals("Invalid number of subresults", 0, resNode.subResults.size());
}
// we're done, close the index reader and the taxonomy.
slowIndexReader.close();
slowTaxoReader.close();
indexDir.close();
taxoDir.close();
IOUtils.close(slowIndexReader, slowTaxoReader, indexDir, taxoDir);
}
/**
@ -245,77 +225,78 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
@Test
public void testGenerationalConsistency() throws Exception {
// Create temporary RAMDirectories
Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
// Create our index/taxonomy writers
IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
// Add a facet to the index
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
addFacets(iParams, indexWriter, taxoWriter, "a", "b");
// Commit Changes
writers[0].indexWriter.commit();
writers[0].taxWriter.commit();
indexWriter.commit();
taxoWriter.commit();
// Open readers
IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
// As this is the first time we have invoked the TotalFacetCountsManager,
// we should expect to compute and not read from disk.
TotalFacetCounts totalCounts =
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
int prevGen = assertRecomputed(totalCounts, 0, "after first attempt to get it!");
// Repeating same operation should pull from the cache - not recomputed.
assertTrue("Should be obtained from cache at 2nd attempt",totalCounts ==
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
TFC.getTotalCounts(indexReader, taxoReader, iParams));
// Repeat the same operation as above. but clear first - now should recompute again
initCache();
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 3rd attempt to get it!");
//store to file
File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
initCache();
TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
TFC.store(outputFile, indexReader, taxoReader, iParams);
totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertRecomputed(totalCounts, prevGen, "after cache clear, 4th attempt to get it!");
//clear and load
initCache();
TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
TFC.load(outputFile, indexReader, taxoReader, iParams);
totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertReadFromDisc(totalCounts, prevGen, "after 5th attempt to get it!");
// Add a new facet to the index, commit and refresh readers
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "c", "d");
writers[0].indexWriter.close();
writers[0].taxWriter.close();
addFacets(iParams, indexWriter, taxoWriter, "c", "d");
IOUtils.close(indexWriter, taxoWriter);
DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
assertNotNull(newTaxoReader);
assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > readers[0].taxReader.getSize());
readers[0].taxReader.close();
readers[0].taxReader = newTaxoReader;
assertTrue("should have received more cagtegories in updated taxonomy", newTaxoReader.getSize() > taxoReader.getSize());
taxoReader.close();
taxoReader = newTaxoReader;
DirectoryReader r2 = DirectoryReader.openIfChanged(readers[0].indexReader);
DirectoryReader r2 = DirectoryReader.openIfChanged(indexReader);
assertNotNull(r2);
readers[0].indexReader.close();
readers[0].indexReader = r2;
indexReader.close();
indexReader = r2;
// now use the new reader - should recompute
totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
prevGen = assertRecomputed(totalCounts, prevGen, "after updating the index - 7th attempt!");
// try again - should not recompute
assertTrue("Should be obtained from cache at 8th attempt",totalCounts ==
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
TFC.getTotalCounts(indexReader, taxoReader, iParams));
readers[0].close();
IOUtils.close(indexReader, taxoReader);
outputFile.delete();
IOUtils.close(dirs[0]);
IOUtils.close(indexDir, taxoDir);
}
private int assertReadFromDisc(TotalFacetCounts totalCounts, int prevGen, String errMsg) {
@ -341,10 +322,12 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
*/
@Test
public void testGrowingTaxonomy() throws Exception {
// Create temporary RAMDirectories
Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(1);
Directory indexDir = newDirectory();
Directory taxoDir = newDirectory();
// Create our index/taxonomy writers
IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
IndexWriter indexWriter = new IndexWriter(indexDir, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
TaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir);
FacetIndexingParams iParams = new FacetIndexingParams() {
@Override
public int getPartitionSize() {
@ -352,37 +335,38 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
}
};
// Add a facet to the index
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
addFacets(iParams, indexWriter, taxoWriter, "a", "b");
// Commit Changes
writers[0].indexWriter.commit();
writers[0].taxWriter.commit();
indexWriter.commit();
taxoWriter.commit();
IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
DirectoryReader indexReader = DirectoryReader.open(indexDir);
TaxonomyReader taxoReader = new DirectoryTaxonomyReader(taxoDir);
// Create TFC and write cache to disk
File outputFile = _TestUtil.createTempFile("test", "tmp", TEMP_DIR);
TFC.store(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
TFC.store(outputFile, indexReader, taxoReader, iParams);
// Make the taxonomy grow without touching the index
for (int i = 0; i < 10; i++) {
writers[0].taxWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
taxoWriter.addCategory(new CategoryPath("foo", Integer.toString(i)));
}
writers[0].taxWriter.commit();
DirectoryTaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(readers[0].taxReader);
taxoWriter.commit();
TaxonomyReader newTaxoReader = TaxonomyReader.openIfChanged(taxoReader);
assertNotNull(newTaxoReader);
readers[0].taxReader.close();
readers[0].taxReader = newTaxoReader;
taxoReader.close();
taxoReader = newTaxoReader;
initCache();
// With the bug, this next call should result in an exception
TFC.load(outputFile, readers[0].indexReader, readers[0].taxReader, iParams);
TotalFacetCounts totalCounts = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
TFC.load(outputFile, indexReader, taxoReader, iParams);
TotalFacetCounts totalCounts = TFC.getTotalCounts(indexReader, taxoReader, iParams);
assertReadFromDisc(totalCounts, 0, "after reading from disk.");
outputFile.delete();
writers[0].close();
readers[0].close();
IOUtils.close(dirs[0]);
IOUtils.close(indexWriter, taxoWriter, indexReader, taxoReader);
IOUtils.close(indexDir, taxoDir);
}
/**
@ -445,46 +429,52 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
*/
@Test
public void testMultipleIndices() throws IOException {
// Create temporary RAMDirectories
Directory[][] dirs = FacetTestUtils.createIndexTaxonomyDirs(2);
Directory indexDir1 = newDirectory(), indexDir2 = newDirectory();
Directory taxoDir1 = newDirectory(), taxoDir2 = newDirectory();
// Create our index/taxonomy writers
IndexTaxonomyWriterPair[] writers = FacetTestUtils.createIndexTaxonomyWriterPair(dirs);
IndexWriter indexWriter1 = new IndexWriter(indexDir1, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
IndexWriter indexWriter2 = new IndexWriter(indexDir2, newIndexWriterConfig(TEST_VERSION_CURRENT, null));
TaxonomyWriter taxoWriter1 = new DirectoryTaxonomyWriter(taxoDir1);
TaxonomyWriter taxoWriter2 = new DirectoryTaxonomyWriter(taxoDir2);
FacetIndexingParams iParams = FacetIndexingParams.ALL_PARENTS;
// Add a facet to the index
addFacets(iParams, writers[0].indexWriter, writers[0].taxWriter, "a", "b");
addFacets(iParams, writers[1].indexWriter, writers[1].taxWriter, "d", "e");
addFacets(iParams, indexWriter1, taxoWriter1, "a", "b");
addFacets(iParams, indexWriter1, taxoWriter1, "d", "e");
// Commit Changes
writers[0].indexWriter.commit();
writers[0].taxWriter.commit();
writers[1].indexWriter.commit();
writers[1].taxWriter.commit();
indexWriter1.commit();
indexWriter2.commit();
taxoWriter1.commit();
taxoWriter2.commit();
// Open two readers
IndexTaxonomyReaderPair[] readers = FacetTestUtils.createIndexTaxonomyReaderPair(dirs);
DirectoryReader indexReader1 = DirectoryReader.open(indexDir1);
DirectoryReader indexReader2 = DirectoryReader.open(indexDir2);
TaxonomyReader taxoReader1 = new DirectoryTaxonomyReader(taxoDir1);
TaxonomyReader taxoReader2 = new DirectoryTaxonomyReader(taxoDir2);
// As this is the first time we have invoked the TotalFacetCountsManager, we
// should expect to compute.
TotalFacetCounts totalCounts0 =
TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
TotalFacetCounts totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
int prevGen = -1;
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 1");
assertTrue("attempt 1b for same input [0] shout find it in cache",
totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
// 2nd Reader - As this is the first time we have invoked the
// TotalFacetCountsManager, we should expect a state of NEW to be returned.
TotalFacetCounts totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
TotalFacetCounts totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 2");
assertTrue("attempt 2b for same input [1] shout find it in cache",
totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
// Right now cache size is one, so first TFC is gone and should be recomputed
totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 3");
// Similarly will recompute the second result
totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
prevGen = assertRecomputed(totalCounts1, prevGen, "after attempt 4");
// Now we set the cache size to two, meaning both should exist in the
@ -492,23 +482,19 @@ public class TestTotalFacetCountsCache extends FacetTestCase {
TFC.setCacheSize(2);
// Re-compute totalCounts0 (was evicted from the cache when the cache was smaller)
totalCounts0 = TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams);
totalCounts0 = TFC.getTotalCounts(indexReader1, taxoReader1, iParams);
prevGen = assertRecomputed(totalCounts0, prevGen, "after attempt 5");
// now both are in the larger cache and should not be recomputed
totalCounts1 = TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams);
totalCounts1 = TFC.getTotalCounts(indexReader2, taxoReader2, iParams);
assertTrue("with cache of size 2 res no. 0 should come from cache",
totalCounts0 == TFC.getTotalCounts(readers[0].indexReader, readers[0].taxReader, iParams));
totalCounts0 == TFC.getTotalCounts(indexReader1, taxoReader1, iParams));
assertTrue("with cache of size 2 res no. 1 should come from cache",
totalCounts1 == TFC.getTotalCounts(readers[1].indexReader, readers[1].taxReader, iParams));
totalCounts1 == TFC.getTotalCounts(indexReader2, taxoReader2, iParams));
writers[0].close();
writers[1].close();
readers[0].close();
readers[1].close();
for (Directory[] dirset : dirs) {
IOUtils.close(dirset);
}
IOUtils.close(indexWriter1, indexWriter2, taxoWriter1, taxoWriter2);
IOUtils.close(indexReader1, indexReader2, taxoReader1, taxoReader2);
IOUtils.close(indexDir1, indexDir2, taxoDir1, taxoDir2);
}
}

14
lucene/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
View File

@ -3,6 +3,7 @@ package org.apache.lucene.facet.search.sampling;
import java.util.List;
import java.util.Random;
import org.apache.lucene.facet.index.params.FacetIndexingParams;
import org.apache.lucene.facet.search.BaseTestTopK;
import org.apache.lucene.facet.search.FacetsAccumulator;
import org.apache.lucene.facet.search.FacetsCollector;
@ -46,8 +47,8 @@ public abstract class BaseSampleTestTopK extends BaseTestTopK {
protected static final int RETRIES = 10;
@Override
protected FacetSearchParams searchParamsWithRequests(int numResults, int partitionSize) {
FacetSearchParams res = super.searchParamsWithRequests(numResults, partitionSize);
protected FacetSearchParams searchParamsWithRequests(int numResults, FacetIndexingParams fip) {
FacetSearchParams res = super.searchParamsWithRequests(numResults, fip);
for (FacetRequest req : res.facetRequests) {
// randomize the way we aggregate results
if (random().nextBoolean()) {
@ -71,20 +72,23 @@ public abstract class BaseSampleTestTopK extends BaseTestTopK {
boolean useRandomSampler = random().nextBoolean();
for (int partitionSize : partitionSizes) {
try {
initIndex(partitionSize);
// complements return counts for all ordinals, so force ALL_PARENTS indexing
// so that it's easier to compare
FacetIndexingParams fip = getFacetIndexingParams(partitionSize, true);
initIndex(fip);
// Get all of the documents and run the query, then do different
// facet counts and compare to control
Query q = new TermQuery(new Term(CONTENT_FIELD, BETA)); // 90% of the docs
ScoredDocIdCollector docCollector = ScoredDocIdCollector.create(indexReader.maxDoc(), false);
FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, partitionSize);
FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, fip);
FacetsCollector fc = FacetsCollector.create(expectedSearchParams, indexReader, taxoReader);
searcher.search(q, MultiCollector.wrap(docCollector, fc));
List<FacetResult> expectedResults = fc.getFacetResults();
FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, partitionSize);
FacetSearchParams samplingSearchParams = searchParamsWithRequests(K, fip);
// try several times in case of failure, because the test has a chance to fail
// if the top K facets are not sufficiently common with the sample set

3
lucene/facet/src/test/org/apache/lucene/facet/search/sampling/OversampleWithDepthTest.java
View File

@ -3,7 +3,6 @@ package org.apache.lucene.facet.search.sampling;
import java.io.IOException;
import java.util.Collections;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.index.FacetFields;
@ -95,7 +94,7 @@ public class OversampleWithDepthTest extends FacetTestCase {
}
private void index100Docs(Directory indexDir, Directory taxoDir, FacetIndexingParams fip) throws IOException {
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer());
IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, null);
IndexWriter w = new IndexWriter(indexDir, iwc);
TaxonomyWriter tw = new DirectoryTaxonomyWriter(taxoDir);

6
lucene/facet/src/test/org/apache/lucene/facet/taxonomy/directory/TestDirectoryTaxonomyReader.java
View File

@ -3,7 +3,7 @@ package org.apache.lucene.facet.taxonomy.directory;
import java.io.IOException;
import java.util.Random;
import org.apache.lucene.analysis.core.KeywordAnalyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.facet.FacetTestCase;
import org.apache.lucene.facet.taxonomy.CategoryPath;
import org.apache.lucene.facet.taxonomy.TaxonomyReader;
@ -256,7 +256,7 @@ public class TestDirectoryTaxonomyReader extends FacetTestCase {
// hold onto IW to forceMerge
// note how we don't close it, since DTW will close it.
final IndexWriter iw = new IndexWriter(dir,
new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(new LogByteSizeMergePolicy()));
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
@Override
@ -299,7 +299,7 @@ public class TestDirectoryTaxonomyReader extends FacetTestCase {
// hold onto IW to forceMerge
// note how we don't close it, since DTW will close it.
final IndexWriter iw = new IndexWriter(dir,
new IndexWriterConfig(TEST_VERSION_CURRENT, new KeywordAnalyzer())
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()))
.setMergePolicy(new LogByteSizeMergePolicy()));
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
@Override

3
lucene/site/xsl/index.xsl
View File

@ -58,7 +58,8 @@
<li><a href="core/overview-summary.html#overview_description">Introduction to Lucene's APIs</a>:
High-level summary of the different Lucene packages. </li>
<li><a href="core/org/apache/lucene/analysis/package-summary.html#package_description">Analysis overview</a>:
Introduction to Lucene's analysis API. </li>
Introduction to Lucene's analysis API. See also the
<a href="core/org/apache/lucene/analysis/TokenStream.html">TokenStream consumer workflow</a>.</li>
</ul>
<h2>Reference Documents</h2>
<ul>

5
lucene/test-framework/src/java/org/apache/lucene/analysis/MockCharFilter.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.analysis;
import java.io.IOException;
import java.io.Reader;
import java.util.Map;
import java.util.SortedMap;
import java.util.TreeMap;
@ -89,8 +90,8 @@ public class MockCharFilter extends CharFilter {
@Override
public int correct(int currentOff) {
SortedMap<Integer,Integer> subMap = corrections.subMap(0, currentOff+1);
int ret = subMap.isEmpty() ? currentOff : currentOff + subMap.get(subMap.lastKey());
Map.Entry<Integer,Integer> lastEntry = corrections.lowerEntry(currentOff+1);
int ret = lastEntry == null ? currentOff : currentOff + lastEntry.getValue();
assert ret >= 0 : "currentOff=" + currentOff + ",diff=" + (ret-currentOff);
return ret;
}

9
solr/CHANGES.txt
View File

@ -95,6 +95,11 @@ Bug Fixes
* SOLR-4342: Fix DataImportHandler stats to be a prper Map (hossman)
* SOLR-3967: langid.enforceSchema option checks source field instead of target field (janhoy)
* SOLR-4380: Replicate after startup option would not replicate until the
IndexWriter was lazily opened. (Mark Miller, Gregg Donovan)
Optimizations
----------------------
@ -115,6 +120,8 @@ Optimizations
* SOLR-4284: Admin UI - make core list scrollable separate from the rest of
the UI (steffkes)
* SOLR-4364: Admin UI - Locale based number formatting (steffkes)
Other Changes
----------------------
@ -123,6 +130,8 @@ Other Changes
* SOLR-4353: Renamed example jetty context file to reduce confusion (hossman)
* SOLR-4384: Make post.jar report timing information (Upayavira via janhoy)
================== 4.1.0 ==================
Versions of Major Components

13
solr/build.xml
View File

@ -560,17 +560,8 @@
<echo message="Checking for broken links..."/>
<check-broken-links dir="${javadoc.dir}"/>
<echo message="Checking for malformed docs..."/>
<!-- TODO: add missing package.htmls and bump this to level=package -->
<check-missing-javadocs dir="${javadoc.dir}" level="none"/>
<!-- prevent the modules without problems from getting worse -->
<check-missing-javadocs dir="${javadoc.dir}/solr-analysis-extras" level="package"/>
<check-missing-javadocs dir="${javadoc.dir}/solr-cell" level="package"/>
<check-missing-javadocs dir="${javadoc.dir}/solr-clustering" level="package"/>
<!-- solr-core: problems: -->
<check-missing-javadocs dir="${javadoc.dir}/solr-dataimporthandler" level="package"/>
<check-missing-javadocs dir="${javadoc.dir}/solr-dataimporthandler-extras" level="package"/>
<check-missing-javadocs dir="${javadoc.dir}/solr-solrj" level="package"/>
<check-missing-javadocs dir="${javadoc.dir}/solr-test-framework" level="package"/>
<!-- TODO: add missing docs for all classes and bump this to level=class -->
<check-missing-javadocs dir="${javadoc.dir}" level="package"/>
</target>
<target name="-ecj-javadoc-lint" depends="documentation,compile-solr-test-framework,-ecj-resolve">

3
solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/schema.xml
View File

@ -23,9 +23,6 @@
kitchen sink thrown in. See example/solr/conf/schema.xml for a
more concise example.
$Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
$Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
$Name: $
-->
<schema name="test" version="1.0">

5
solr/contrib/extraction/src/test-files/extraction/solr/collection1/conf/solrconfig.xml
View File

@ -17,11 +17,6 @@
limitations under the License.
-->
<!-- $Id: solrconfig.xml 382610 2006-03-03 01:43:03Z yonik $
$Source$
$Name$
-->
<config>
<luceneMatchVersion>${tests.luceneMatchVersion:LUCENE_CURRENT}</luceneMatchVersion>
<jmx />

16
solr/contrib/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java
View File

@ -222,10 +222,6 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro
log.debug("Mapping field "+fieldName+" using document global language "+fieldLang);
}
String mappedOutputField = getMappedField(fieldName, fieldLang);
if(enforceSchema && schema.getFieldOrNull(fieldName) == null) {
log.warn("Unsuccessful field name mapping to {}, field does not exist, skipping mapping.", mappedOutputField, fieldName);
mappedOutputField = fieldName;
}
if (mappedOutputField != null) {
log.debug("Mapping field {} to {}", doc.getFieldValue(docIdField), fieldLang);
@ -350,17 +346,23 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro
/**
* Returns the name of the field to map the current contents into, so that they are properly analyzed. For instance
* if the currentField is "text" and the code is "en", the new field would be "text_en". If such a field doesn't exist,
* then null is returned.
* if the currentField is "text" and the code is "en", the new field would by default be "text_en".
* This method also performs custom regex pattern replace if configured. If enforceSchema=true
* and the resulting field name doesn't exist, then null is returned.
*
* @param currentField The current field name
* @param language the language code
* @return The new schema field name, based on pattern and replace
* @return The new schema field name, based on pattern and replace, or null if illegal
*/
protected String getMappedField(String currentField, String language) {
String lc = lcMap.containsKey(language) ? lcMap.get(language) : language;
String newFieldName = langPattern.matcher(mapPattern.matcher(currentField).replaceFirst(mapReplaceStr)).replaceFirst(lc);
if(enforceSchema && schema.getFieldOrNull(newFieldName) == null) {
log.warn("Unsuccessful field name mapping from {} to {}, field does not exist and enforceSchema=true; skipping mapping.", currentField, newFieldName);
return null;
} else {
log.debug("Doing mapping from "+currentField+" with language "+language+" to field "+newFieldName);
}
return newFieldName;
}

3
solr/contrib/langid/src/test-files/langid/solr/collection1/conf/schema.xml
View File

@ -23,9 +23,6 @@
kitchen sink thrown in. See example/solr/conf/schema.xml for a
more concise example.
$Id: schema.xml 382610 2006-03-03 01:43:03Z yonik $
$Source: /cvs/main/searching/solr-configs/test/WEB-INF/classes/schema.xml,v $
$Name: $
-->
<schema name="test" version="1.5">

16
solr/contrib/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java
View File

@ -93,7 +93,7 @@ public abstract class LanguageIdentifierUpdateProcessorFactoryTestCase extends S
parameters = new ModifiableSolrParams();
parameters.add("langid.fl", "name");
parameters.add("langid.map.lcmap", "jp:s zh:cjk ko:cjk");
parameters.add("langid.enforceSchema", "true");
parameters.set("langid.enforceSchema", "false");
liProcessor = createLangIdProcessor(parameters);
assertEquals("test_no", liProcessor.getMappedField("test", "no"));
@ -102,13 +102,17 @@ public abstract class LanguageIdentifierUpdateProcessorFactoryTestCase extends S
assertEquals("test_cjk", liProcessor.getMappedField("test", "zh"));
assertEquals("test_cjk", liProcessor.getMappedField("test", "ko"));
// Prove support for other mapping regex
parameters.add("langid.map.pattern", "text_(.*?)_field");
parameters.add("langid.map.replace", "$1_{lang}Text");
// Test that enforceSchema correctly catches illegal field and returns null
parameters.set("langid.enforceSchema", "true");
liProcessor = createLangIdProcessor(parameters);
assertEquals(null, liProcessor.getMappedField("inputfield", "sv"));
assertEquals("title_noText", liProcessor.getMappedField("text_title_field", "no"));
assertEquals("body_svText", liProcessor.getMappedField("text_body_field", "sv"));
// Prove support for other mapping regex, still with enforceSchema=true
parameters.add("langid.map.pattern", "text_(.*?)_field");
parameters.add("langid.map.replace", "$1_{lang}_s");
liProcessor = createLangIdProcessor(parameters);
assertEquals("title_no_s", liProcessor.getMappedField("text_title_field", "no"));
assertEquals("body_sv_s", liProcessor.getMappedField("text_body_field", "sv"));
}
@Test

30
solr/core/src/java/org/apache/solr/analysis/package.html Normal file
View File

@ -0,0 +1,30 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Factories and classes specific to text analysis and the creation of {@link org.apache.lucene.analysis.TokenStream}s
</p>
<p>
See {@link org.apache.lucene.analysis} for additional details.
</p>
</body>
</html>

30
solr/core/src/java/org/apache/solr/client/solrj/embedded/package.html Normal file
View File

@ -0,0 +1,30 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
SolrJ client implementations for embedded solr access.
</p>
<p>
See {@link org.apache.solr.client.solrj} for additional details.
</p>
</body>
</html>

27
solr/core/src/java/org/apache/solr/cloud/package.html Normal file
View File

@ -0,0 +1,27 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Classes for dealing with ZooKeeper when operating in <a href="http://wiki.apache.org/solr/SolrCloud">SolrCloud</a> mode.
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/common/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Commonly reused classes and interfaces (deprecated package, do not add new classes)
</p>
</body>
</html>

27
solr/core/src/java/org/apache/solr/core/package.html Normal file
View File

@ -0,0 +1,27 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Core classes implementin Solr internals and the management of {@link org.apache.solr.core.SolrCore}s
</p>
</body>
</html>

7
solr/core/src/java/org/apache/solr/handler/ReplicationHandler.java
View File

@ -42,6 +42,7 @@ import org.apache.commons.io.IOUtils;
import org.apache.lucene.index.IndexCommit;
import org.apache.lucene.index.IndexDeletionPolicy;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@ -877,9 +878,9 @@ public class ReplicationHandler extends RequestHandlerBase implements SolrCoreAw
}
}
// reboot the writer on the new index
// TODO: perhaps this is no longer necessary then?
// core.getUpdateHandler().newIndexWriter(true);
// ensure the writer is init'd so that we have a list of commit points
RefCounted<IndexWriter> iw = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
iw.decref();
} catch (IOException e) {
LOG.warn("Unable to get IndexCommit on startup", e);

27
solr/core/src/java/org/apache/solr/handler/admin/package.html Normal file
View File

@ -0,0 +1,27 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
{@link org.apache.solr.request.SolrRequestHandler} implementations for powering he Solr Admin UI
</p>
</body>
</html>

28
solr/core/src/java/org/apache/solr/handler/component/package.html Normal file
View File

@ -0,0 +1,28 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
{@link org.apache.solr.handler.component.SearchComponent} implementations for
use in {@link org.apache.solr.handler.component.SearchHandler}
</p>
</body>
</html>

28
solr/core/src/java/org/apache/solr/handler/loader/package.html Normal file
View File

@ -0,0 +1,28 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
{@link org.apache.solr.handler.loader.ContentStreamLoader} implementations for
use in {@link org.apache.solr.handler.ContentStreamHandlerBase} implementations
</p>
</body>
</html>

27
solr/core/src/java/org/apache/solr/handler/package.html Normal file
View File

@ -0,0 +1,27 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Concrete implementations of {@link org.apache.solr.request.SolrRequestHandler}
</p>
</body>
</html>

30
solr/core/src/java/org/apache/solr/highlight/package.html Normal file
View File

@ -0,0 +1,30 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
{@link org.apache.solr.highlight.SolrHighlighter} API and related implementaions and utilities
</p>
<p>
See {@link org.apache.lucene.search.highlight} for addition information.
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/internal/csv/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Internal classes used for reading/writing CSV
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/internal/csv/writer/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Internal classes used for reading/writing CSV
</p>
</body>
</html>

27
solr/core/src/java/org/apache/solr/logging/jul/package.html Normal file
View File

@ -0,0 +1,27 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
JUL based implementation of {@link org.apache.solr.logging.LogWatcher}
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/logging/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
APIs related to capturing log event info in the {@link org.apache.solr.handler.admin.LoggingHandler}
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Commonly reused classes and interfaces (deprecated package, do not add new classes)
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/parser/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Solr native variant of the {@linkplain org.apache.lucene.queryparser.classic.QueryParser Lucene Classic QueryParser}
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/request/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
APIs and classes for dealing with Solr requests
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/response/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
API and implementations of {@link org.apache.solr.response.QueryResponseWriter} for formating Solr request responses
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/response/transform/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
APIs and implementations of {@link org.apache.solr.response.transform.DocTransformer} for modifying documents in Solr request responses
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/schema/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
{@link org.apache.solr.schema.IndexSchema} and {@link org.apache.solr.schema.FieldType} implementations for powering schema.xml
</p>
</body>
</html>

27
solr/core/src/java/org/apache/solr/search/function/distance/package.html Normal file
View File

@ -0,0 +1,27 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Solr implementations of {@link org.apache.lucene.queries.function.ValueSource} for distance based function queries.
</p>
</body>
</html>

27
solr/core/src/java/org/apache/solr/search/function/package.html Normal file
View File

@ -0,0 +1,27 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Solr implementations of {@link org.apache.lucene.queries.function.ValueSource} for function queries.
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/search/grouping/collector/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Grouping related {@link org.apache.lucene.search.Collector}s
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/search/grouping/distributed/command/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Internal classes used to implement distributed result grouping
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/search/grouping/distributed/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Internal APIs for distribute result grouping
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/search/grouping/distributed/requestfactory/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Internal classes used to implement distributed result grouping
</p>
</body>
</html>

29
solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/package.html Normal file
View File

@ -0,0 +1,29 @@
<!doctype html public "-//w3c//dtd html 4.0 transitional//en">
<!--
Licensed to the Apache Software Foundation (ASF) under one or more
contributor license agreements. See the NOTICE file distributed with
this work for additional information regarding copyright ownership.
The ASF licenses this file to You under the Apache License, Version 2.0
(the "License"); you may not use this file except in compliance with
the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-->
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
</head>
<body>
<p>
Internal classes used to implement distributed result grouping
</p>
</body>
</html>

Some files were not shown because too many files have changed in this diff Show More