Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

SOLR-2452: Merged up to trunk r1126232; adjusted solr/config/analysis-extras/ directory structure to match other solr contribs; updated configuration for maven and intellij idea. 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/solr2452@1126234 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Steven Rowe 2011-05-22 21:45:19 +00:00
parent dc58134cdc 1a936ed69d
commit 674feedec6
874 changed files with 76434 additions and 11035 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
README.txt
View File

@ -7,6 +7,7 @@ modules/ is shared code
To compile the sources run 'ant compile'
To run all the tests run 'ant test'
To setup your ide run 'ant idea' or 'ant eclipse'
For Maven info, see dev-tools/maven/README.maven.
For more information on how to contribute see:
http://wiki.apache.org/lucene-java/HowToContribute

4
build.xml
View File

@ -19,7 +19,7 @@
<project name="lucene-solr" default="test" basedir=".">
<import file="common-build.xml"/>
<target name="test" description="Test both Lucene and Solr">
<target name="test" description="Test both Lucene and Solr" depends="validate">
<sequential>
<subant target="test" inheritall="false" failonerror="true">
<fileset dir="lucene" includes="build.xml" />
@ -35,7 +35,7 @@
<fileset dir="solr" includes="build.xml" />
</subant></sequential>
</target>
<target name="compile" depends="validate" description="Compile Lucene and Solr">
<target name="compile" description="Compile Lucene and Solr">
<sequential>
<subant target="compile" inheritall="false" failonerror="true">

11
dev-tools/eclipse/dot.classpath
View File

@ -53,6 +53,8 @@
<classpathentry kind="src" path="modules/analysis/stempel/src/test"/>
<classpathentry kind="src" path="modules/benchmark/src/java"/>
<classpathentry kind="src" path="modules/benchmark/src/test"/>
<classpathentry kind="src" path="modules/grouping/src/java"/>
<classpathentry kind="src" path="modules/grouping/src/test"/>
<classpathentry kind="src" path="solr/src/java"/>
<classpathentry kind="src" path="solr/src/webapp/src"/>
<classpathentry kind="src" path="solr/src/common"/>
@ -95,7 +97,7 @@
<classpathentry kind="lib" path="modules/benchmark/lib/commons-digester-1.7.jar"/>
<classpathentry kind="lib" path="modules/benchmark/lib/commons-logging-1.0.4.jar"/>
<classpathentry kind="lib" path="modules/benchmark/lib/xercesImpl-2.9.1-patched-XERCESJ-1257.jar"/>
<classpathentry kind="lib" path="solr/lib/apache-solr-noggit-r944541.jar"/>
<classpathentry kind="lib" path="solr/lib/apache-solr-noggit-r1099557.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-beanutils-1.7.0.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-codec-1.4.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-collections-3.2.1.jar"/>
@ -105,7 +107,6 @@
<classpathentry kind="lib" path="solr/lib/commons-io-1.4.jar"/>
<classpathentry kind="lib" path="solr/lib/commons-lang-2.4.jar"/>
<classpathentry kind="lib" path="solr/lib/easymock-2.2.jar"/>
<classpathentry kind="lib" path="solr/lib/geronimo-stax-api_1.0_spec-1.0.1.jar"/>
<classpathentry kind="lib" path="solr/lib/guava-r05.jar"/>
<classpathentry kind="lib" path="solr/lib/jcl-over-slf4j-1.6.1.jar"/>
<classpathentry kind="lib" path="solr/lib/junit-4.7.jar"/>
@ -124,13 +125,13 @@
<classpathentry kind="lib" path="solr/example/lib/jsp-2.1/jsp-2.1-glassfish-2.1.v20091210.jar"/>
<classpathentry kind="lib" path="solr/example/lib/jsp-2.1/jsp-2.1-jetty-6.1.26.jar"/>
<classpathentry kind="lib" path="solr/example/lib/jsp-2.1/jsp-api-2.1-glassfish-2.1.v20091210.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/carrot2-core-3.4.2.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/hppc-0.3.1.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/carrot2-core-3.5.0.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/hppc-0.3.3.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/jackson-core-asl-1.5.2.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/jackson-mapper-asl-1.5.2.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/mahout-collections-0.3.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/mahout-math-0.3.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/simple-xml-2.3.5.jar"/>
<classpathentry kind="lib" path="solr/contrib/clustering/lib/simple-xml-2.4.1.jar"/>
<classpathentry kind="lib" path="solr/contrib/dataimporthandler/lib/activation-1.1.jar"/>
<classpathentry kind="lib" path="solr/contrib/dataimporthandler/lib/mail-1.4.1.jar"/>
<classpathentry kind="lib" path="solr/contrib/extraction/lib/asm-3.1.jar"/>

2
dev-tools/idea/.idea/ant.xml
View File

@ -26,10 +26,12 @@
<buildFile url="file://$PROJECT_DIR$/modules/analysis/smartcn/build.xml" />
<buildFile url="file://$PROJECT_DIR$/modules/analysis/stempel/build.xml" />
<buildFile url="file://$PROJECT_DIR$/modules/benchmark/build.xml" />
<buildFile url="file://$PROJECT_DIR$/modules/grouping/build.xml" />
<buildFile url="file://$PROJECT_DIR$/solr/build.xml" />
<buildFile url="file://$PROJECT_DIR$/solr/contrib/analysis-extras/build.xml" />
<buildFile url="file://$PROJECT_DIR$/solr/contrib/clustering/build.xml" />
<buildFile url="file://$PROJECT_DIR$/solr/contrib/dataimporthandler/build.xml" />
<buildFile url="file://$PROJECT_DIR$/solr/contrib/dataimporthandler-extras/build.xml" />
<buildFile url="file://$PROJECT_DIR$/solr/contrib/extraction/build.xml" />
<buildFile url="file://$PROJECT_DIR$/solr/contrib/uima/build.xml" />
</component>

1
dev-tools/idea/.idea/compiler.xml
View File

@ -27,6 +27,7 @@
<entry name="?*.tbl" />
<entry name="?*.tld" />
<entry name="?*.txt" />
<entry name="?*.utf8"/>
<entry name="?*.xml" />
<entry name="?*.xsl" />
<entry name="?*.vm" />

3
dev-tools/idea/.idea/modules.xml
View File

@ -26,11 +26,12 @@
<module filepath="$PROJECT_DIR$/modules/analysis/smartcn/smartcn.iml" />
<module filepath="$PROJECT_DIR$/modules/analysis/stempel/stempel.iml" />
<module filepath="$PROJECT_DIR$/modules/benchmark/benchmark.iml" />
<module filepath="$PROJECT_DIR$/modules/grouping/grouping.iml" />
<module filepath="$PROJECT_DIR$/solr/solr.iml" />
<module filepath="$PROJECT_DIR$/solr/contrib/analysis-extras/analysis-extras.iml" />
<module filepath="$PROJECT_DIR$/solr/contrib/clustering/clustering.iml" />
<module filepath="$PROJECT_DIR$/solr/contrib/dataimporthandler/dataimporthandler.iml" />
<module filepath="$PROJECT_DIR$/solr/contrib/dataimporthandler/src/extras/extras.iml" />
<module filepath="$PROJECT_DIR$/solr/contrib/dataimporthandler-extras/dataimporthandler-extras.iml" />
<module filepath="$PROJECT_DIR$/solr/contrib/extraction/extraction.iml" />
<module filepath="$PROJECT_DIR$/solr/contrib/uima/uima.iml" />
</modules>

68
dev-tools/idea/.idea/workspace.xml
View File

@ -4,7 +4,7 @@
<configuration default="false" name="analysis-extras contrib" type="JUnit" factoryName="JUnit">
<module name="analysis-extras" />
<option name="TEST_OBJECT" value="package" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/contrib/analysis-extras/build" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/build/contrib/analysis-extras" />
<option name="VM_PARAMETERS" value="-ea -Dtests.luceneMatchVersion=4.0 -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
@ -39,7 +39,7 @@
<configuration default="false" name="clustering contrib" type="JUnit" factoryName="JUnit">
<module name="clustering" />
<option name="TEST_OBJECT" value="package" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/contrib/clustering/build" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/build/contrib/clustering" />
<option name="VM_PARAMETERS" value="-ea -Dtests.luceneMatchVersion=4.0 -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
@ -53,21 +53,28 @@
<configuration default="false" name="dataimporthandler contrib" type="JUnit" factoryName="JUnit">
<module name="dataimporthandler" />
<option name="TEST_OBJECT" value="package" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/contrib/dataimporthandler/target" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/build/contrib/dataimporthandler" />
<option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
<configuration default="false" name="dataimporthandler-extras contrib" type="JUnit" factoryName="JUnit">
<module name="dataimporthandler-extras" />
<option name="TEST_OBJECT" value="package" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/build/contrib/dataimporthandler-extras" />
<option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
<configuration default="false" name="extraction contrib" type="JUnit" factoryName="JUnit">
<module name="extraction" />
<option name="TEST_OBJECT" value="package" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/contrib/extraction/build" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/build/contrib/extraction" />
<option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
<configuration default="false" name="extras from dataimporthandler contrib" type="JUnit" factoryName="JUnit">
<module name="extras" />
<configuration default="false" name="grouping module" type="JUnit" factoryName="JUnit">
<module name="grouping" />
<option name="TEST_OBJECT" value="package" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/contrib/dataimporthandler/target/extras" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/modules/grouping/build" />
<option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
@ -186,7 +193,7 @@
<configuration default="false" name="uima contrib" type="JUnit" factoryName="JUnit">
<module name="uima" />
<option name="TEST_OBJECT" value="package" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/contrib/uima/build" />
<option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$/solr/build/contrib/uima" />
<option name="VM_PARAMETERS" value="-ea -Dtests.luceneMatchVersion=4.0 -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
@ -204,7 +211,7 @@
<option name="VM_PARAMETERS" value="-ea -DtempDir=temp" />
<option name="TEST_SEARCH_SCOPE"><value defaultName="singleModule" /></option>
</configuration>
<list size="29">
<list size="30">
<item index="0" class="java.lang.String" itemvalue="JUnit.analysis-extras contrib" />
<item index="1" class="java.lang.String" itemvalue="JUnit.ant contrib" />
<item index="2" class="java.lang.String" itemvalue="JUnit.bdb contrib" />
@ -213,27 +220,28 @@
<item index="5" class="java.lang.String" itemvalue="JUnit.clustering contrib" />
<item index="6" class="java.lang.String" itemvalue="JUnit.common analysis module" />
<item index="7" class="java.lang.String" itemvalue="JUnit.dataimporthandler contrib" />
<item index="8" class="java.lang.String" itemvalue="JUnit.extraction contrib" />
<item index="9" class="java.lang.String" itemvalue="JUnit.extras from dataimporthandler contrib" />
<item index="10" class="java.lang.String" itemvalue="JUnit.highlighter contrib" />
<item index="11" class="java.lang.String" itemvalue="JUnit.icu analysis module" />
<item index="12" class="java.lang.String" itemvalue="JUnit.instantiated contrib" />
<item index="13" class="java.lang.String" itemvalue="JUnit.lucene" />
<item index="14" class="java.lang.String" itemvalue="JUnit.lucli contrib" />
<item index="15" class="java.lang.String" itemvalue="JUnit.memory contrib" />
<item index="16" class="java.lang.String" itemvalue="JUnit.misc contrib" />
<item index="17" class="java.lang.String" itemvalue="JUnit.phonetic analysis module" />
<item index="18" class="java.lang.String" itemvalue="JUnit.queries contrib" />
<item index="19" class="java.lang.String" itemvalue="JUnit.queryparser contrib" />
<item index="20" class="java.lang.String" itemvalue="JUnit.smartcn analysis module" />
<item index="21" class="java.lang.String" itemvalue="JUnit.solr" />
<item index="22" class="java.lang.String" itemvalue="JUnit.spatial contrib" />
<item index="23" class="java.lang.String" itemvalue="JUnit.spellchecker contrib" />
<item index="24" class="java.lang.String" itemvalue="JUnit.stempel analysis module" />
<item index="25" class="java.lang.String" itemvalue="JUnit.swing contrib" />
<item index="26" class="java.lang.String" itemvalue="JUnit.uima contrib" />
<item index="27" class="java.lang.String" itemvalue="JUnit.wordnet contrib" />
<item index="28" class="java.lang.String" itemvalue="JUnit.xml-query-parser contrib" />
<item index="8" class="java.lang.String" itemvalue="JUnit.dataimporthandler-extras contrib" />
<item index="9" class="java.lang.String" itemvalue="JUnit.extraction contrib" />
<item index="10" class="java.lang.String" itemvalue="JUnit.grouping module" />
<item index="11" class="java.lang.String" itemvalue="JUnit.highlighter contrib" />
<item index="12" class="java.lang.String" itemvalue="JUnit.icu analysis module" />
<item index="13" class="java.lang.String" itemvalue="JUnit.instantiated contrib" />
<item index="14" class="java.lang.String" itemvalue="JUnit.lucene" />
<item index="15" class="java.lang.String" itemvalue="JUnit.lucli contrib" />
<item index="16" class="java.lang.String" itemvalue="JUnit.memory contrib" />
<item index="17" class="java.lang.String" itemvalue="JUnit.misc contrib" />
<item index="18" class="java.lang.String" itemvalue="JUnit.phonetic analysis module" />
<item index="19" class="java.lang.String" itemvalue="JUnit.queries contrib" />
<item index="20" class="java.lang.String" itemvalue="JUnit.queryparser contrib" />
<item index="21" class="java.lang.String" itemvalue="JUnit.smartcn analysis module" />
<item index="22" class="java.lang.String" itemvalue="JUnit.solr" />
<item index="23" class="java.lang.String" itemvalue="JUnit.spatial contrib" />
<item index="24" class="java.lang.String" itemvalue="JUnit.spellchecker contrib" />
<item index="25" class="java.lang.String" itemvalue="JUnit.stempel analysis module" />
<item index="26" class="java.lang.String" itemvalue="JUnit.swing contrib" />
<item index="27" class="java.lang.String" itemvalue="JUnit.uima contrib" />
<item index="28" class="java.lang.String" itemvalue="JUnit.wordnet contrib" />
<item index="29" class="java.lang.String" itemvalue="JUnit.xml-query-parser contrib" />
</list>
</component>
</project>

8
dev-tools/idea/lucene/contrib/ant/ant.iml
View File

@ -9,10 +9,6 @@
<sourceFolder url="file://$MODULE_DIR$/src/resources" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -27,5 +23,9 @@
</orderEntry>
<orderEntry type="library" name="Ant" level="project" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/lucene/contrib/db/bdb-je/bdb-je.iml
View File

@ -8,8 +8,6 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -23,5 +21,7 @@
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/lucene/contrib/db/bdb/bdb.iml
View File

@ -8,8 +8,6 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -23,5 +21,7 @@
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

14
dev-tools/idea/lucene/contrib/demo/demo.iml
View File

@ -8,20 +8,10 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
<library>
<CLASSES>
<root url="file://$MODULE_DIR$/lib" />
</CLASSES>
<JAVADOC />
<SOURCES />
<jarDirectory url="file://$MODULE_DIR$/lib" recursive="false" />
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

8
dev-tools/idea/lucene/contrib/highlighter/highlighter.iml
View File

@ -8,12 +8,12 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="memory" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="memory" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

6
dev-tools/idea/lucene/contrib/instantiated/instantiated.iml
View File

@ -8,11 +8,11 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

8
dev-tools/idea/lucene/contrib/lucli/lucli.iml
View File

@ -8,10 +8,6 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -24,5 +20,9 @@
<jarDirectory url="file://$MODULE_DIR$/lib" recursive="false" />
</library>
</orderEntry>
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

6
dev-tools/idea/lucene/contrib/memory/memory.iml
View File

@ -8,11 +8,11 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/lucene/contrib/misc/misc.iml
View File

@ -8,10 +8,10 @@
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/lucene/contrib/queries/queries.iml
View File

@ -8,8 +8,6 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -22,5 +20,7 @@
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

6
dev-tools/idea/lucene/contrib/queryparser/queryparser.iml
View File

@ -9,11 +9,11 @@
<sourceFolder url="file://$MODULE_DIR$/src/resources" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

6
dev-tools/idea/lucene/contrib/spatial/spatial.iml
View File

@ -8,12 +8,12 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

8
dev-tools/idea/lucene/contrib/spellchecker/spellchecker.iml
View File

@ -8,12 +8,12 @@
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

8
dev-tools/idea/lucene/contrib/swing/swing.iml
View File

@ -8,12 +8,12 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

8
dev-tools/idea/lucene/contrib/wordnet/wordnet.iml
View File

@ -8,12 +8,12 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

8
dev-tools/idea/lucene/contrib/xml-query-parser/xml-query-parser.iml
View File

@ -10,13 +10,13 @@
<sourceFolder url="file://$MODULE_DIR$/src/demo/WebContent" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="library" name="Servlet API 2.4" level="project" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

2
dev-tools/idea/modules/analysis/common/common.iml
View File

@ -10,9 +10,9 @@
<sourceFolder url="file://$MODULE_DIR$/src/java/tools" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/modules/analysis/icu/icu.iml
View File

@ -10,8 +10,6 @@
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/resources" isTestSource="false" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -25,5 +23,7 @@
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/modules/analysis/phonetic/phonetic.iml
View File

@ -8,8 +8,6 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -23,5 +21,7 @@
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/modules/analysis/smartcn/smartcn.iml
View File

@ -9,10 +9,10 @@
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/resources" isTestSource="false" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

4
dev-tools/idea/modules/analysis/stempel/stempel.iml
View File

@ -9,10 +9,10 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/resources" isTestSource="false" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

14
dev-tools/idea/modules/benchmark/benchmark.iml
View File

@ -10,13 +10,6 @@
<excludeFolder url="file://$MODULE_DIR$/temp" />
<excludeFolder url="file://$MODULE_DIR$/work" />
</content>
<orderEntry type="module" module-name="highlighter" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="icu" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="memory" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -30,5 +23,12 @@
</library>
</orderEntry>
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="highlighter" />
<orderEntry type="module" module-name="icu" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="memory" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

16
dev-tools/idea/modules/grouping/grouping.iml Normal file
View File

@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/build/classes/java" />
<output-test url="file://$MODULE_DIR$/build/classes/test" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

22
dev-tools/idea/solr/contrib/analysis-extras/analysis-extras.iml
View File

@ -1,21 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/build/classes" />
<output-test url="file://$MODULE_DIR$/build/test-classes" />
<output url="file://$MODULE_DIR$/../../build/contrib/analysis-extras/classes" />
<output-test url="file://$MODULE_DIR$/../../build/contrib/analysis-extras/test-classes" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/build" />
<excludeFolder url="file://$MODULE_DIR$/lucene-libs" />
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="icu" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="smartcn" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="stempel" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -30,5 +22,11 @@
</orderEntry>
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="icu" />
<orderEntry type="module" module-name="smartcn" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="stempel" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

25
dev-tools/idea/solr/contrib/clustering/clustering.iml
View File

@ -1,25 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/build/classes" />
<output-test url="file://$MODULE_DIR$/build/test-classes" />
<output url="file://$MODULE_DIR$/../../build/contrib/clustering/classes" />
<output-test url="file://$MODULE_DIR$/../../build/contrib/clustering/test-classes" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/build" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="highlighter" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="memory" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="phonetic" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="spatial" />
<orderEntry type="module" module-name="spellchecker" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="module-library">
@ -35,5 +24,15 @@
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" name="Solr DIH library" level="project" />
<orderEntry type="library" scope="TEST" name="JUnit" level="project" />
<orderEntry type="module" module-name="highlighter" />
<orderEntry type="module" module-name="memory" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="phonetic" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="spatial" />
<orderEntry type="module" module-name="spellchecker" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="common" />
</component>
</module>

16
dev-tools/idea/solr/contrib/dataimporthandler/src/extras/extras.iml → dev-tools/idea/solr/contrib/dataimporthandler-extras/dataimporthandler-extras.iml
View File

@ -1,22 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/../../target/extras/classes" />
<output-test url="file://$MODULE_DIR$/../../target/extras/test-classes" />
<output url="file://$MODULE_DIR$/../../build/contrib/dataimporthandler-extras/classes" />
<output-test url="file://$MODULE_DIR$/../../build/contrib/dataimporthandler-extras/test-classes" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/test/resources" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" />
</content>
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="dataimporthandler" />
<orderEntry type="module" module-name="lucene" scope="TEST" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" name="Solr DIH library" level="project" />
<orderEntry type="library" name="Solr extraction library" level="project" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="dataimporthandler" />
<orderEntry type="module" module-name="lucene" scope="TEST" />
</component>
</module>

11
dev-tools/idea/solr/contrib/dataimporthandler/dataimporthandler.iml
View File

@ -1,22 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/target/classes" />
<output-test url="file://$MODULE_DIR$/target/test-classes" />
<output url="file://$MODULE_DIR$/../../build/contrib/dataimporthandler/classes" />
<output-test url="file://$MODULE_DIR$/../../build/contrib/dataimporthandler/test-classes" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/webapp" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/target" />
</content>
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" name="Solr DIH library" level="project" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
</component>
</module>

9
dev-tools/idea/solr/contrib/extraction/extraction.iml
View File

@ -1,20 +1,19 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/build/classes" />
<output-test url="file://$MODULE_DIR$/build/test-classes" />
<output url="file://$MODULE_DIR$/../../build/contrib/extraction/classes" />
<output-test url="file://$MODULE_DIR$/../../build/contrib/extraction/test-classes" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/build" />
</content>
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="lucene" scope="TEST" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" name="Solr extraction library" level="project" />
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="lucene" scope="TEST" />
</component>
</module>

9
dev-tools/idea/solr/contrib/uima/uima.iml
View File

@ -1,18 +1,15 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="JAVA_MODULE" version="4">
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_6" inherit-compiler-output="false">
<output url="file://$MODULE_DIR$/build/classes" />
<output-test url="file://$MODULE_DIR$/build/test-classes" />
<output url="file://$MODULE_DIR$/../../build/contrib/uima/classes" />
<output-test url="file://$MODULE_DIR$/../../build/contrib/uima/test-classes" />
<exclude-output />
<content url="file://$MODULE_DIR$">
<sourceFolder url="file://$MODULE_DIR$/src/test/java" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test/resources" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/main/resources" isTestSource="false" />
<excludeFolder url="file://$MODULE_DIR$/build" />
</content>
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="lucene" scope="TEST" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Solr library" level="project" />
@ -26,5 +23,7 @@
<jarDirectory url="file://$MODULE_DIR$/lib" recursive="false" />
</library>
</orderEntry>
<orderEntry type="module" module-name="solr" />
<orderEntry type="module" module-name="lucene" scope="TEST" />
</component>
</module>

12
dev-tools/idea/solr/solr.iml
View File

@ -12,23 +12,23 @@
<sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
<sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test-files" isTestSource="true" />
<sourceFolder url="file://$MODULE_DIR$/src/test-framework" isTestSource="true" />
<excludeFolder url="file://$MODULE_DIR$/build" />
<excludeFolder url="file://$MODULE_DIR$/dist" />
<excludeFolder url="file://$MODULE_DIR$/lucene-libs" />
<excludeFolder url="file://$MODULE_DIR$/package" />
</content>
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" name="Solr example library" level="project" />
<orderEntry type="module" module-name="spatial" />
<orderEntry type="module" module-name="highlighter" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="icu" />
<orderEntry type="module" module-name="queries" />
<orderEntry type="module" module-name="misc" />
<orderEntry type="module" module-name="phonetic" />
<orderEntry type="module" module-name="spellchecker" />
<orderEntry type="module" module-name="common" />
<orderEntry type="module" module-name="lucene" />
<orderEntry type="inheritedJdk" />
<orderEntry type="sourceFolder" forTests="false" />
<orderEntry type="library" name="Solr library" level="project" />
<orderEntry type="library" name="Solr example library" level="project" />
</component>
</module>

131
dev-tools/maven/README.maven Normal file
View File

@ -0,0 +1,131 @@
====================================
Lucene/Solr Maven build instructions
====================================
Contents:
A. How to use nightly Jenkins-built Lucene/Solr Maven artifacts
B. How to generate Lucene Maven artifacts
C. How to generate Solr Maven artifacts
D. How to use Maven to build Lucene/Solr
-----
A. How to use nightly Jenkins-built Lucene/Solr Maven artifacts
The most recently produced nightly Jenkins-built Lucene and Solr Maven
artifacts are available in Maven repository layout here:
<https://builds.apache.org/hudson/job/Lucene-Solr-Maven-trunk/lastSuccessfulBuild/artifact/maven_artifacts/>
B. How to generate Lucene Maven artifacts
1. Prerequisites: JDK 1.5+, Ant 1.7.X, and maven-ant-tasks-2.1.1.jar
In order to generate Maven artifacts for Lucene/Solr, you must first
download the Maven ant tasks JAR (maven-ant-tasks-2.1.1.jar), e.g.
from <http://maven.apache.org/ant-tasks/download.html>, and add it
to any one of the following:
a. Your $HOME/.ant/lib/ directory (C:\Users\username\.ant\lib\ under
Windows Vista/7); or
b. Your $ANT_HOME/lib/ directory (%ANT_HOME%\lib\ under Windows); or
c. Your $CLASSPATH (%CLASSPATH% under Windows); or
d. Your ant commond line: "-lib /path/to/maven-ant-tasks-2.1.1.jar".
2. Run the following command from the lucene/ directory:
ant generate-maven-artifacts
The above command will create an internal Maven repository under
lucene/dist/maven/, including POMs, binary .jars, source .jars,
and javadoc .jars, for Lucene Core, for the Lucene test framework,
for each contrib, and for each module under the top-level modules/
directory.
C. How to generate Solr Maven artifacts
1. Prerequisites: JDK 1.6+; Ant 1.7.X; and maven-ant-tasks-2.1.1.jar
(see item A.1. above for where to put the Maven ant tasks jar).
2. Run the following from the solr/ directory:
ant generate-maven-artifacts
The above command will create an internal Maven repository under
solr/package/maven/, including POMs, binary .jars, source .jars,
and javadoc .jars, for Solr Core, for the Solr test framework,
for each contrib, and for the Solr .war (for which there are no
source or javadoc .jars).
D. How to use Maven to build Lucene/Solr
In summary, to enable Maven builds, perform the following:
svn update
ant get-maven-poms
mvn -N -Pbootstrap install
The details, followed by some example Maven commands:
1. Prerequisites: JDK 1.5+ (for Lucene); JDK 1.6+ (for Solr);
Maven 2.2.1 or 3.0.X
2. Make sure your sources are up to date. If you checked your sources out
from the Apache Subversion repository, run "svn update" from the top
level.
3. Copy the Maven POM templates from under dev-tools/maven/ to where they
they need to go in order to drive the Maven build, using the following
command from the top-level directory:
ant get-maven-poms
Note that you will need to do this whenever changes to the POM
templates are committed. It's a good idea to follow every "svn update"
with "ant get-maven-poms" for this reason.
The above command copies all of the POM templates from dev-tools/maven/,
filling in the project version with the default "X.X-SNAPSHOT". If you
want the POMs and the Maven-built artifacts to have a version other than
the default, you can supply an alternate version on the command line
with the above command, e.g.:
ant -Dversion=4.0-my-special-version get-maven-poms
4. Populate your local repository with .jars & POMs for dependencies that
are not available from public Maven repositories (a.k.a. "non-mavenized
dependencies"):
mvn -N -Pbootstrap install
Note that you will need to do this whenever changes to the non-Mavenized
dependencies are committed. It's a good idea to follow every
"svn update" with "ant get-maven-poms" and "mvn -N -Pbootstrap install"
for this reason.
Some example Maven commands you can use after you perform the above
preparatory steps:
- Compile, package, and install all artifacts to your local repository:
mvn install
After compiling and packaging, but before installing each module's
artifact, the above command will also run all the module's tests.
To compile, package and install all artifacts without running any tests:
mvn -DskipTests install
- Run tests:
mvn test
To run all test methods defined in a test class:
mvn -Dtest=TestClassName test

71
dev-tools/maven/modules/grouping/pom.xml.template Normal file
View File

@ -0,0 +1,71 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-parent</artifactId>
<version>@version@</version>
<relativePath>../../lucene/pom.xml</relativePath>
</parent>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-grouping</artifactId>
<packaging>jar</packaging>
<name>Lucene Grouping</name>
<description>Lucene Grouping Module</description>
<properties>
<module-directory>modules/grouping</module-directory>
<build-directory>build</build-directory>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>lucene-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<directory>${build-directory}</directory>
<outputDirectory>${build-directory}/classes/java</outputDirectory>
<testOutputDirectory>${build-directory}/classes/test</testOutputDirectory>
<sourceDirectory>src/java</sourceDirectory>
<testSourceDirectory>src/test</testSourceDirectory>
<testResources>
<testResource>
<directory>${project.build.testSourceDirectory}</directory>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</testResource>
</testResources>
</build>
</project>

1
dev-tools/maven/modules/pom.xml.template
View File

@ -33,6 +33,7 @@
<modules>
<module>analysis</module>
<module>benchmark</module>
<module>grouping</module>
</modules>
<build>
<directory>build/lucene-modules-aggregator</directory>

23
dev-tools/maven/pom.xml.template
View File

@ -24,6 +24,7 @@
<groupId>org.apache</groupId>
<artifactId>apache</artifactId>
<version>8</version>
<relativePath/>
</parent>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-solr-grandparent</artifactId>
@ -105,14 +106,6 @@
</license>
</licenses>
<repositories>
<repository>
<id>carrot2.org</id>
<name>Carrot2 Maven2 repository</name>
<url>http://download.carrot2.org/maven2/</url>
<snapshots>
<updatePolicy>never</updatePolicy>
</snapshots>
</repository>
<repository>
<id>apache.snapshots</id>
<name>Apache Snapshot Repository</name>
@ -132,6 +125,11 @@
<artifactId>lucene-icu4j</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
<version>r05</version>
</dependency>
<dependency>
<groupId>com.sleepycat</groupId>
<artifactId>berkeleydb</artifactId>
@ -232,11 +230,6 @@
<artifactId>commons-compress</artifactId>
<version>1.1</version>
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
<version>1.0.1</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-commons-csv</artifactId>
@ -300,7 +293,7 @@
<dependency>
<groupId>org.carrot2</groupId>
<artifactId>carrot2-core</artifactId>
<version>3.4.2</version>
<version>3.5.0</version>
</dependency>
<dependency>
<groupId>org.codehaus.woodstox</groupId>
@ -694,7 +687,7 @@
<artifactId>solr-noggit</artifactId>
<version>${project.version}</version>
<packaging>jar</packaging>
<file>solr/lib/apache-solr-noggit-r944541.jar</file>
<file>solr/lib/apache-solr-noggit-r1099557.jar</file>
</configuration>
</execution>
<execution>

9
dev-tools/maven/solr/contrib/analysis-extras/pom.xml.template
View File

@ -33,7 +33,7 @@
<description>Apache Solr Analysis Extras</description>
<properties>
<module-directory>solr/contrib/analysis-extras</module-directory>
<build-directory>build</build-directory>
<build-directory>../../build/contrib/analysis-extras</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
@ -44,9 +44,8 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<artifactId>solr-test-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
@ -95,8 +94,8 @@
<directory>${build-directory}</directory>
<outputDirectory>${build-directory}/classes</outputDirectory>
<testOutputDirectory>${build-directory}/test-classes</testOutputDirectory>
<sourceDirectory>src/java</sourceDirectory>
<testSourceDirectory>src/test</testSourceDirectory>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<testResources>
<testResource>
<directory>src/test-files</directory>

5
dev-tools/maven/solr/contrib/clustering/pom.xml.template
View File

@ -33,7 +33,7 @@
<description>Apache Solr Clustering</description>
<properties>
<module-directory>solr/contrib/clustering</module-directory>
<build-directory>build</build-directory>
<build-directory>../../build/contrib/clustering</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
@ -44,9 +44,8 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<artifactId>solr-test-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>

23
dev-tools/maven/solr/contrib/dataimporthandler/src/extras/pom.xml.template → dev-tools/maven/solr/contrib/dataimporthandler-extras/pom.xml.template
View File

@ -24,7 +24,7 @@
<groupId>org.apache.solr</groupId>
<artifactId>solr-parent</artifactId>
<version>@version@</version>
<relativePath>../../../../pom.xml</relativePath>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.solr</groupId>
<artifactId>solr-dataimporthandler-extras</artifactId>
@ -32,8 +32,8 @@
<name>Apache Solr DataImportHandler Extras</name>
<description>Apache Solr DataImportHandler Extras</description>
<properties>
<module-directory>solr/contrib/dataimporthandler/src/extras</module-directory>
<build-directory>../../target/extras</build-directory>
<module-directory>solr/contrib/dataimporthandler-extras</module-directory>
<build-directory>../../build/contrib/dataimporthandler-extras</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
@ -44,9 +44,8 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<artifactId>solr-test-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
@ -104,16 +103,16 @@
</dependencies>
<build>
<directory>${build-directory}</directory>
<outputDirectory>${build-directory}/extras/classes</outputDirectory>
<testOutputDirectory>${build-directory}/extras/test-classes</testOutputDirectory>
<sourceDirectory>main/java</sourceDirectory>
<testSourceDirectory>test/java</testSourceDirectory>
<outputDirectory>${build-directory}/classes</outputDirectory>
<testOutputDirectory>${build-directory}/test-classes</testOutputDirectory>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<testResources>
<testResource>
<directory>test/resources</directory>
<directory>src/test/resources</directory>
</testResource>
<testResource>
<directory>../../../../src/test-files</directory>
<directory>../../src/test-files</directory>
</testResource>
</testResources>
<plugins>
@ -122,7 +121,7 @@
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<java.util.logging.config.file>../../../../../../testlogging.properties</java.util.logging.config.file>
<java.util.logging.config.file>../../../../testlogging.properties</java.util.logging.config.file>
</systemPropertyVariables>
</configuration>
</plugin>

110
dev-tools/maven/solr/contrib/dataimporthandler/pom.xml.template
View File

@ -27,22 +27,110 @@
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.solr</groupId>
<artifactId>solr-dataimporthandler-aggregator</artifactId>
<packaging>pom</packaging>
<name>Apache Solr DataImportHandler aggregator POM</name>
<description>Apache Solr DataImportHandler aggregator POM</description>
<modules>
<module>src</module>
<module>src/extras</module>
</modules>
<artifactId>solr-dataimporthandler</artifactId>
<packaging>jar</packaging>
<name>Apache Solr DataImportHandler</name>
<description>Apache Solr DataImportHandler</description>
<properties>
<module-directory>solr/contrib/dataimporthandler</module-directory>
<build-directory>../../build/contrib/dataimporthandler</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-solrj</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<directory>target/solr-dataimporthandler-aggregator</directory>
<directory>${build-directory}</directory>
<outputDirectory>${build-directory}/classes</outputDirectory>
<testOutputDirectory>${build-directory}/test-classes</testOutputDirectory>
<sourceDirectory>src/main/java</sourceDirectory>
<testSourceDirectory>src/test/java</testSourceDirectory>
<testResources>
<testResource>
<directory>src/test/resources</directory>
</testResource>
<testResource>
<directory>../../src/test-files</directory>
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skip>true</skip>
<systemPropertyVariables>
<java.util.logging.config.file>../../../../testlogging.properties</java.util.logging.config.file>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>

143
dev-tools/maven/solr/contrib/dataimporthandler/src/pom.xml.template
View File

@ -1,143 +0,0 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.solr</groupId>
<artifactId>solr-parent</artifactId>
<version>@version@</version>
<relativePath>../../../pom.xml</relativePath>
</parent>
<groupId>org.apache.solr</groupId>
<artifactId>solr-dataimporthandler</artifactId>
<packaging>jar</packaging>
<name>Apache Solr DataImportHandler</name>
<description>Apache Solr DataImportHandler</description>
<properties>
<module-directory>solr/contrib/dataimporthandler</module-directory>
<build-directory>../target</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-solrj</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
</dependency>
<dependency>
<groupId>commons-io</groupId>
<artifactId>commons-io</artifactId>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.easymock</groupId>
<artifactId>easymock</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.mortbay.jetty</groupId>
<artifactId>jetty-util</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<directory>${build-directory}</directory>
<outputDirectory>${build-directory}/classes</outputDirectory>
<testOutputDirectory>${build-directory}/test-classes</testOutputDirectory>
<sourceDirectory>main/java</sourceDirectory>
<testSourceDirectory>test/java</testSourceDirectory>
<testResources>
<testResource>
<directory>test/resources</directory>
</testResource>
<testResource>
<directory>../../../src/test-files</directory>
</testResource>
</testResources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<systemPropertyVariables>
<java.util.logging.config.file>../../../../../testlogging.properties</java.util.logging.config.file>
</systemPropertyVariables>
</configuration>
</plugin>
</plugins>
</build>
</project>

5
dev-tools/maven/solr/contrib/extraction/pom.xml.template
View File

@ -36,7 +36,7 @@
</description>
<properties>
<module-directory>solr/contrib/extraction</module-directory>
<build-directory>build</build-directory>
<build-directory>../../build/contrib/extraction</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
@ -47,9 +47,8 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<artifactId>solr-test-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>

1
dev-tools/maven/solr/contrib/pom.xml.template
View File

@ -34,6 +34,7 @@
<module>analysis-extras</module>
<module>clustering</module>
<module>dataimporthandler</module>
<module>dataimporthandler-extras</module>
<module>extraction</module>
<module>uima</module>
</modules>

5
dev-tools/maven/solr/contrib/uima/pom.xml.template
View File

@ -33,7 +33,7 @@
<description>Apache Solr - UIMA integration</description>
<properties>
<module-directory>solr/contrib/uima</module-directory>
<build-directory>build</build-directory>
<build-directory>../../build/contrib/uima</build-directory>
<tests.luceneMatchVersion>4.0</tests.luceneMatchVersion>
</properties>
<dependencies>
@ -44,9 +44,8 @@
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<artifactId>solr-test-framework</artifactId>
<version>${project.version}</version>
<type>test-jar</type>
<scope>test</scope>
</dependency>
<dependency>

1
dev-tools/maven/solr/pom.xml.template
View File

@ -35,6 +35,7 @@
<module>src</module>
<module>src/solrj</module>
<module>src/webapp</module>
<module>src/test-framework</module>
<module>contrib</module>
</modules>
<properties>

37
dev-tools/maven/solr/src/pom.xml.template
View File

@ -96,10 +96,6 @@
<groupId>org.apache.solr</groupId>
<artifactId>solr-commons-csv</artifactId>
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
</dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
@ -156,6 +152,10 @@
<artifactId>servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@ -197,17 +197,6 @@
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
<executions>
<execution>
<goals>
<goal>test-jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
@ -248,6 +237,24 @@
</programs>
</configuration>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
<id>add-test-source</id>
<phase>generate-test-sources</phase>
<goals>
<goal>add-test-source</goal>
</goals>
<configuration>
<sources>
<source>test-framework</source>
</sources>
</configuration>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

17
dev-tools/maven/solr/src/solrj/pom.xml.template
View File

@ -42,22 +42,12 @@
<artifactId>lucene-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-analyzers-common</artifactId>
<version>${project.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.geronimo.specs</groupId>
<artifactId>geronimo-stax-api_1.0_spec</artifactId>
</dependency>
<dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
@ -88,15 +78,10 @@
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<directory>${build-directory}</directory>
<outputDirectory>${build-directory}</outputDirectory>
<outputDirectory>${build-directory}/classes</outputDirectory>
<sourceDirectory>.</sourceDirectory>
<testResources/>
<plugins>

76
dev-tools/maven/solr/src/test-framework/pom.xml.template Normal file
View File

@ -0,0 +1,76 @@
<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.solr</groupId>
<artifactId>solr-parent</artifactId>
<version>@version@</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.solr</groupId>
<artifactId>solr-test-framework</artifactId>
<packaging>jar</packaging>
<name>Apache Solr Test Framework</name>
<description>Apache Solr Test Framework</description>
<properties>
<module-directory>solr/src/test-framework</module-directory>
<build-directory>../../build</build-directory>
</properties>
<dependencies>
<dependency>
<groupId>${project.groupId}</groupId>
<artifactId>solr-core</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-test-framework</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
</dependency>
</dependencies>
<build>
<directory>${build-directory}</directory>
<outputDirectory>${build-directory}/classes/test-framework</outputDirectory>
<sourceDirectory>.</sourceDirectory>
<resources>
<resource>
<directory>.</directory>
<excludes>
<exclude>**/*.java</exclude>
</excludes>
</resource>
</resources>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<configuration>
<skip>true</skip>
</configuration>
</plugin>
</plugins>
</build>
</project>

61
dev-tools/scripts/diffSources.py Normal file
View File

@ -0,0 +1,61 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import subprocess
import sys
# recursive, unified output format, treat missing files as present but empty
DIFF_FLAGS = '-ruN'
if '-skipWhitespace' in sys.argv:
sys.argv.remove('-skipWhitespace')
# ignores only whitespace changes
DIFF_FLAGS += 'bBw'
if len(sys.argv) != 3:
print
print 'Usage: python -u diffSources.py <dir1> <dir2> [-skipWhitespace]'
print
print '''This tool creates an applying patch between two directories.
While you could use this to make a committable patch from a branch, that approach loses
the svn history from the branch (better to use "svn merge --reintegrate", for example). This
diff output should not be considered "authoritative" from a merging standpoint as it does
not reflect what svn will do on merge.
'''
print
sys.exit(0)
p = subprocess.Popen(['diff', DIFF_FLAGS, '-x', '.svn', '-x', 'build', sys.argv[1], sys.argv[2]], shell=False, stdout=subprocess.PIPE)
keep = False
while True:
l = p.stdout.readline()
if l == '':
break
if l.endswith('\r\n'):
l = l[:-2]
elif l.endswith('\n'):
l = l[:-1]
if l.startswith('diff ') or l.startswith('Binary files '):
keep = l.lower().find('/build/') == -1 and (l.lower().startswith('Only in') or ((l.lower().endswith('.java') or l.lower().endswith('.txt') or l.lower().endswith('.xml') or l.lower().endswith('.iml')) and l.find('/.svn/') == -1))
if keep:
print
print
print l.strip()
elif keep:
print l
elif l.startswith('Only in'):
print l.strip()

227
lucene/CHANGES.txt
View File

@ -141,6 +141,11 @@ Changes in backwards compatibility policy
* LUCENE-2315: AttributeSource's methods for accessing attributes are now final,
else its easy to corrupt the internal states. (Uwe Schindler)
* LUCENE-2814: The IndexWriter.flush method no longer takes "boolean
flushDocStores" argument, as we now always flush doc stores (index
files holding stored fields and term vectors) while flushing a
segment. (Mike McCandless)
Changes in Runtime Behavior
* LUCENE-2846: omitNorms now behaves like omitTermFrequencyAndPositions, if you
@ -157,18 +162,76 @@ Changes in Runtime Behavior
* LUCENE-2720: IndexWriter throws IndexFormatTooOldException on open, rather
than later when e.g. a merge starts. (Shai Erera, Mike McCandless, Uwe Schindler)
* LUCENE-1076: The default merge policy is now able to merge
non-contiguous segments, which means docIDs no longer necessarily
say "in order". If this is a problem then you can use either of the
LogMergePolicy impls, and call setRequireContiguousMerge(true).
(Mike McCandless)
* LUCENE-2881: FieldInfos is now tracked per segment. Before it was tracked
per IndexWriter session, which resulted in FieldInfos that had the FieldInfo
properties from all previous segments combined. Field numbers are now tracked
globally across IndexWriter sessions and persisted into a X.fnx file on
successful commit. The corresponding file format changes are backwards-
compatible. (Michael Busch, Simon Willnauer)
* LUCENE-2956, LUCENE-2573, LUCENE-2324, LUCENE-2555: Changes from
DocumentsWriterPerThread:
- IndexWriter now uses a DocumentsWriter per thread when indexing documents.
Each DocumentsWriterPerThread indexes documents in its own private segment,
and the in memory segments are no longer merged on flush. Instead, each
segment is separately flushed to disk and subsequently merged with normal
segment merging.
- DocumentsWriterPerThread (DWPT) is now flushed concurrently based on a
FlushPolicy. When a DWPT is flushed, a fresh DWPT is swapped in so that
indexing may continue concurrently with flushing. The selected
DWPT flushes all its RAM resident documents do disk. Note: Segment flushes
don't flush all RAM resident documents but only the documents private to
the DWPT selected for flushing.
- Flushing is now controlled by FlushPolicy that is called for every add,
update or delete on IndexWriter. By default DWPTs are flushed either on
maxBufferedDocs per DWPT or the global active used memory. Once the active
memory exceeds ramBufferSizeMB only the largest DWPT is selected for
flushing and the memory used by this DWPT is substracted from the active
memory and added to a flushing memory pool, which can lead to temporarily
higher memory usage due to ongoing indexing.
- IndexWriter now can utilize ramBufferSize > 2048 MB. Each DWPT can address
up to 2048 MB memory such that the ramBufferSize is now bounded by the max
number of DWPT avaliable in the used DocumentsWriterPerThreadPool.
IndexWriters net memory consumption can grow far beyond the 2048 MB limit if
the applicatoin can use all available DWPTs. To prevent a DWPT from
exhausting its address space IndexWriter will forcefully flush a DWPT if its
hard memory limit is exceeded. The RAMPerThreadHardLimitMB can be controlled
via IndexWriterConfig and defaults to 1945 MB.
Since IndexWriter flushes DWPT concurrently not all memory is released
immediately. Applications should still use a ramBufferSize significantly
lower than the JVMs avaliable heap memory since under high load multiple
flushing DWPT can consume substantial transient memory when IO performance
is slow relative to indexing rate.
- IndexWriter#commit now doesn't block concurrent indexing while flushing all
'currently' RAM resident documents to disk. Yet, flushes that occur while a
a full flush is running are queued and will happen after all DWPT involved
in the full flush are done flushing. Applications using multiple threads
during indexing and trigger a full flush (eg call commmit() or open a new
NRT reader) can use significantly more transient memory.
- IndexWriter#addDocument and IndexWriter.updateDocument can block indexing
threads if the number of active + number of flushing DWPT exceed a
safety limit. By default this happens if 2 * max number available thread
states (DWPTPool) is exceeded. This safety limit prevents applications from
exhausting their available memory if flushing can't keep up with
concurrently indexing threads.
- IndexWriter only applies and flushes deletes if the maxBufferedDelTerms
limit is reached during indexing. No segment flushes will be triggered
due to this setting.
- IndexWriter#flush(boolean, boolean) doesn't synchronized on IndexWriter
anymore. A dedicated flushLock has been introduced to prevent multiple full-
flushes happening concurrently.
- DocumentsWriter doesn't write shared doc stores anymore.
(Mike McCandless, Michael Busch, Simon Willnauer)
API Changes
@ -267,9 +330,6 @@ New features
* LUCENE-2489: Added PerFieldCodecWrapper (in oal.index.codecs) which
lets you set the Codec per field (Mike McCandless)
* LUCENE-1810: Added FieldSelectorResult.LATENT to not cache lazy loaded fields
(Tim Smith, Grant Ingersoll)
* LUCENE-2373: Extend CodecProvider to use SegmentInfosWriter and
SegmentInfosReader to allow customization of SegmentInfos data.
(Andrzej Bialecki)
@ -299,9 +359,6 @@ New features
use MultiFields static methods directly, instead) if you need to use
the flex APIs directly on a composite reader. (Mike McCandless)
* LUCENE-2692: Added several new SpanQuery classes for positional checking
(match is in a range, payload is a specific value) (Grant Ingersoll)
* LUCENE-2690: MultiTermQuery boolean rewrites per segment.
(Uwe Schindler, Robert Muir, Mike McCandless, Simon Willnauer)
@ -333,10 +390,14 @@ New features
* LUCENE-2862: Added TermsEnum.totalTermFreq() and
Terms.getSumTotalTermFreq(). (Mike McCandless, Robert Muir)
* LUCENE-3001: Added TrieFieldHelper to write solr compatible numeric
fields without the solr dependency. (ryan)
* LUCENE-3003: Added new expert class oal.index.DocTermsOrd,
refactored from Solr's UnInvertedField, for accessing term ords for
multi-valued fields, per document. This is similar to FieldCache in
that it inverts the index to compute the ords, but differs in that
it's able to handle multi-valued fields and does not hold the term
bytes in RAM. (Mike McCandless)
Optimizations
* LUCENE-2588: Don't store unnecessary suffixes when writing the terms
@ -366,24 +427,6 @@ Bug fixes
with more document deletions is requested before a reader with fewer
deletions, provided they share some segments. (yonik)
* LUCENE-2936: PhraseQuery score explanations were not correctly
identifying matches vs non-matches. (hossman)
* LUCENE-2996: addIndexes(IndexReader) did not flush before adding the new
indexes, causing existing deletions to be applied on the incoming indexes as
well. (Shai Erera, Mike McCandless)
Test Cases
* LUCENE-3002: added 'tests.iter.min' to control 'tests.iter' by allowing to
stop iterating if at least 'tests.iter.min' ran and a failure occured.
(Shai Erera, Chris Hostetter)
Build
* LUCENE-3006: Building javadocs will fail on warnings by default. Override with -Dfailonjavadocwarning=false (sarowe, gsingers)
======================= Lucene 3.x (not yet released) =======================
Changes in backwards compatibility policy
@ -393,11 +436,117 @@ Changes in backwards compatibility policy
a method getHeapArray() was added to retrieve the internal heap array as a
non-generic Object[]. (Uwe Schindler, Yonik Seeley)
* LUCENE-1076: IndexWriter.setInfoStream now throws IOException
(Mike McCandless, Shai Erera)
* LUCENE-3084: MergePolicy.OneMerge.segments was changed from
SegmentInfos to a List<SegmentInfo>. SegmentInfos itsself was changed
to no longer extend Vector<SegmentInfo> (to update code that is using
Vector-API, use the new asList() and asSet() methods returning unmodifiable
collections; modifying SegmentInfos is now only possible through
the explicitely declared methods). IndexWriter.segString() now takes
Iterable<SegmentInfo> instead of List<SegmentInfo>. A simple recompile
should fix this. MergePolicy and SegmentInfos are internal/experimental
APIs not covered by the strict backwards compatibility policy.
(Uwe Schindler, Mike McCandless)
Changes in runtime behavior
* LUCENE-3065: When a NumericField is retrieved from a Document loaded
from IndexReader (or IndexSearcher), it will now come back as
NumericField not as a Field with a string-ified version of the
numeric value you had indexed. Note that this only applies for
newly-indexed Documents; older indices will still return Field
with the string-ified numeric value. If you call Document.get(),
the value comes still back as String, but Document.getFieldable()
returns NumericField instances. (Uwe Schindler, Ryan McKinley,
Mike McCandless)
* LUCENE-1076: Changed the default merge policy from
LogByteSizeMergePolicy to TieredMergePolicy, as of Version.LUCENE_32
(passed to IndexWriterConfig), which is able to merge non-contiguous
segments. This means docIDs no longer necessarily stay "in order"
during indexing. If this is a problem then you can use either of
the LogMergePolicy impls. (Mike McCandless)
New features
* LUCENE-3082: Added index upgrade tool oal.index.IndexUpgrader
that allows to upgrade all segments to last recent supported index
format without fully optimizing. (Uwe Schindler, Mike McCandless)
* LUCENE-1076: Added TieredMergePolicy which is able to merge non-contiguous
segments, which means docIDs no longer necessarily stay "in order".
(Mike McCandless, Shai Erera)
* LUCENE-3071: Adding ReversePathHierarchyTokenizer, added skip parameter to
PathHierarchyTokenizer (Olivier Favre via ryan)
* LUCENE-1421, LUCENE-3102: added CachingCollector which allow you to cache
document IDs and scores encountered during the search, and "replay" them to
another Collector. (Mike McCandless, Shai Erera)
API Changes
* LUCENE-3061: IndexWriter's getNextMerge() and merge(OneMerge) are now public
(though @lucene.experimental), allowing for custom MergeScheduler
implementations. (Shai Erera)
* LUCENE-3065: Document.getField() was deprecated, as it throws
ClassCastException when loading lazy fields or NumericFields.
(Uwe Schindler, Ryan McKinley, Mike McCandless)
* LUCENE-2027: Directory.touchFile is deprecated and will be removed
in 4.0. (Mike McCandless)
Optimizations
* LUCENE-2990: ArrayUtil/CollectionUtil.*Sort() methods now exit early
on empty or one-element lists/arrays. (Uwe Schindler)
* LUCENE-2897: Apply deleted terms while flushing a segment. We still
buffer deleted terms to later apply to past segments. (Mike McCandless)
Bug fixes
* LUCENE-2996: addIndexes(IndexReader) did not flush before adding the new
indexes, causing existing deletions to be applied on the incoming indexes as
well. (Shai Erera, Mike McCandless)
* LUCENE-3024: Index with more than 2.1B terms was hitting AIOOBE when
seeking TermEnum (eg used by Solr's faceting) (Tom Burton-West, Mike
McCandless)
* LUCENE-3042: When a filter or consumer added Attributes to a TokenStream
chain after it was already (partly) consumed [or clearAttributes(),
captureState(), cloneAttributes(),... was called by the Tokenizer],
the Tokenizer calling clearAttributes() or capturing state after addition
may not do this on the newly added Attribute. This bug affected only
very special use cases of the TokenStream-API, most users would not
have recognized it. (Uwe Schindler, Robert Muir)
* LUCENE-3054: PhraseQuery can in some cases stack overflow in
SorterTemplate.quickSort(). This fix also adds an optimization to
PhraseQuery as term with lower doc freq will also have less positions.
(Uwe Schindler, Robert Muir, Otis Gospodnetic)
* LUCENE-3068: sloppy phrase query failed to match valid documents when multiple
query terms had same position in the query. (Doron Cohen)
* LUCENE-3012: Lucene writes the header now for separate norm files (*.sNNN)
(Robert Muir)
Build
* LUCENE-3006: Building javadocs will fail on warnings by default.
Override with -Dfailonjavadocwarning=false (sarowe, gsingers)
Test Cases
* LUCENE-3002: added 'tests.iter.min' to control 'tests.iter' by allowing to
stop iterating if at least 'tests.iter.min' ran and a failure occured.
(Shai Erera, Chris Hostetter)
======================= Lucene 3.1.0 =======================
Changes in backwards compatibility policy
@ -936,6 +1085,12 @@ New features
* LUCENE-2913: Add missing getters to Numeric* classes. (Uwe Schindler)
* LUCENE-1810: Added FieldSelectorResult.LATENT to not cache lazy loaded fields
(Tim Smith, Grant Ingersoll)
* LUCENE-2692: Added several new SpanQuery classes for positional checking
(match is in a range, payload is a specific value) (Grant Ingersoll)
Optimizations
* LUCENE-2494: Use CompletionService in ParallelMultiSearcher instead of
@ -1379,6 +1534,10 @@ Bug fixes
that warming is free to do whatever it needs to. (Earwin Burrfoot
via Mike McCandless)
* LUCENE-3029: Fix corner case when MultiPhraseQuery is used with zero
position-increment tokens that would sometimes assign different
scores to identical docs. (Mike McCandless)
* LUCENE-2486: Fixed intermittent FileNotFoundException on doc store
files when a mergedSegmentWarmer is set on IndexWriter. (Mike
McCandless)

8
lucene/MIGRATE.txt
View File

@ -312,6 +312,8 @@ LUCENE-1458, LUCENE-2111: Flexible Indexing
- o.a.l.analysis.ReusableAnalyzerBase -> o.a.l.analysis.util.ReusableAnalyzerBase
- o.a.l.analysis.StopwordAnalyzerBase -> o.a.l.analysis.util.StopwordAnalyzerBase
- o.a.l.analysis.WordListLoader -> o.a.l.analysis.util.WordListLoader
- o.a.l.analysis.CharTokenizer -> o.a.l.analysis.util.CharTokenizer
- o.a.l.util.CharacterUtils -> o.a.l.analysis.util.CharacterUtils
* LUCENE-2514: The option to use a Collator's order (instead of binary order) for
sorting and range queries has been moved to contrib/queries.
@ -356,3 +358,9 @@ LUCENE-1458, LUCENE-2111: Flexible Indexing
field as a parameter, this is removed due to the fact the entire Similarity (all methods)
can now be configured per-field.
Methods that apply to the entire query such as coord() and queryNorm() exist in SimilarityProvider.
* LUCENE-1076: TieredMergePolicy is now the default merge policy.
It's able to merge non-contiguous segments; this may cause problems
for applications that rely on Lucene's internal document ID
assigment. If so, you should instead use LogByteSize/DocMergePolicy
during indexing.

7
lucene/common-build.xml
View File

@ -73,6 +73,7 @@
</condition>
<property name="tests.multiplier" value="1" />
<property name="tests.codec" value="randomPerField" />
<property name="tests.codecprovider" value="random" />
<property name="tests.locale" value="random" />
<property name="tests.timezone" value="random" />
<property name="tests.directory" value="random" />
@ -308,7 +309,7 @@
</copy>
</target>
<target name="compile" depends="compile-core, validate-lucene">
<target name="compile" depends="compile-core">
<!-- convenience target to compile core -->
</target>
@ -499,6 +500,8 @@
<sysproperty key="tests.verbose" value="${tests.verbose}"/>
<!-- set the codec tests should run with -->
<sysproperty key="tests.codec" value="${tests.codec}"/>
<!-- set the codec provider tests should run with -->
<sysproperty key="tests.codecprovider" value="${tests.codecprovider}"/>
<!-- set the locale tests should run with -->
<sysproperty key="tests.locale" value="${tests.locale}"/>
<!-- set the timezone tests should run with -->
@ -568,7 +571,7 @@
</sequential>
</macrodef>
<target name="test" depends="compile-test,junit-mkdir,junit-sequential,junit-parallel" description="Runs unit tests"/>
<target name="test" depends="compile-test,validate-lucene,junit-mkdir,junit-sequential,junit-parallel" description="Runs unit tests"/>
<target name="junit-mkdir">
<mkdir dir="${junit.output.dir}"/>

64
lucene/contrib/CHANGES.txt
View File

@ -42,10 +42,68 @@ API Changes
Instead, use SimilarityProvider to return different SweetSpotSimilaritys
for different fields, this way all parameters (such as TF factors) can be
customized on a per-field basis. (Robert Muir)
Bug Fixes
* LUCENE-3045: fixed QueryNodeImpl.containsTag(String key) that was
not lowercasing the key before checking for the tag (Adriano Crestani)
======================= Lucene 3.x (not yet released) =======================
(No changes)
Changes in runtime behavior
* LUCENE-3086: ItalianAnalyzer now uses ElisionFilter with a set of Italian
contractions by default. (Robert Muir)
Bug Fixes
* LUCENE-3045: fixed QueryNodeImpl.containsTag(String key) that was
not lowercasing the key before checking for the tag (Adriano Crestani)
* LUCENE-3026: SmartChineseAnalyzer's WordTokenFilter threw NullPointerException
on sentences longer than 32,767 characters. (wangzhenghang via Robert Muir)
* LUCENE-2939: Highlighter should try and use maxDocCharsToAnalyze in
WeightedSpanTermExtractor when adding a new field to MemoryIndex as well as
when using CachingTokenStream. This can be a significant performance bug for
large documents. (Mark Miller)
* LUCENE-3043: GermanStemmer threw IndexOutOfBoundsException if it encountered
a zero-length token. (Robert Muir)
* LUCENE-3044: ThaiWordFilter didn't reset its cached state correctly, this only
caused a problem if you consumed a tokenstream, then reused it, added different
attributes to it, and consumed it again. (Robert Muir, Uwe Schindler)
* LUCENE-3113: Fixed some minor analysis bugs: double-reset() in ReusableAnalyzerBase
and ShingleAnalyzerWrapper, missing end() implementations in PrefixAwareTokenFilter
and PrefixAndSuffixAwareTokenFilter, invocations of incrementToken() after it
already returned false in CommonGramsQueryFilter, HyphenatedWordsFilter,
ShingleFilter, and SynonymsFilter. (Robert Muir, Steven Rowe, Uwe Schindler)
New Features
* LUCENE-3016: Add analyzer for Latvian. (Robert Muir)
* LUCENE-1421: create new grouping contrib module, enabling search
results to be grouped by a single-valued indexed field. This
module was factored out of Solr's grouping implementation, but
it cannot group by function queries nor arbitrary queries. (Mike
McCandless)
* LUCENE-3098: add AllGroupsCollector, to collect all unique groups
(but in unspecified order). (Martijn van Groningen via Mike
McCandless)
* LUCENE-3092: Added NRTCachingDirectory in contrib/misc, which
caches small segments in RAM. This is useful, in the near-real-time
case where the indexing rate is lowish but the reopen rate is
highish, to take load off the IO system. (Mike McCandless)
Optimizations
* LUCENE-3040: Switch all analysis consumers (highlighter, morelikethis, memory, ...)
over to reusableTokenStream(). (Robert Muir)
======================= Lucene 3.1.0 =======================
@ -156,6 +214,10 @@ Bug fixes
* LUCENE-2943: Fix thread-safety issues with ICUCollationKeyFilter.
(Robert Muir)
* LUCENE-3087: Highlighter: fix case that was preventing highlighting
of exact phrase when tokens overlap. (Pierre Gossé via Mike
McCandless)
API Changes
* LUCENE-2867: Some contrib queryparser methods that receives CharSequence as

8
lucene/contrib/ant/src/java/org/apache/lucene/ant/IndexTask.java
View File

@ -39,7 +39,7 @@ import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.IndexSearcher;
@ -285,9 +285,9 @@ public class IndexTask extends Task {
IndexWriterConfig conf = new IndexWriterConfig(
Version.LUCENE_CURRENT, analyzer).setOpenMode(
create ? OpenMode.CREATE : OpenMode.APPEND);
LogMergePolicy lmp = (LogMergePolicy) conf.getMergePolicy();
lmp.setUseCompoundFile(useCompoundIndex);
lmp.setMergeFactor(mergeFactor);
TieredMergePolicy tmp = (TieredMergePolicy) conf.getMergePolicy();
tmp.setUseCompoundFile(useCompoundIndex);
tmp.setMaxMergeAtOnce(mergeFactor);
IndexWriter writer = new IndexWriter(dir, conf);
int totalFiles = 0;
int totalIndexed = 0;

8
lucene/contrib/ant/src/test/org/apache/lucene/ant/HtmlDocumentTest.java
View File

@ -17,8 +17,6 @@ package org.apache.lucene.ant;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.ant.DocumentTestCase;
import org.apache.lucene.ant.HtmlDocument;
@ -27,7 +25,8 @@ public class HtmlDocumentTest extends DocumentTestCase
HtmlDocument doc;
@Override
public void setUp() throws IOException {
public void setUp() throws Exception {
super.setUp();
doc = new HtmlDocument(getFile("test.html"));
}
@ -37,8 +36,9 @@ public class HtmlDocumentTest extends DocumentTestCase
}
@Override
public void tearDown() {
public void tearDown() throws Exception {
doc = null;
super.tearDown();
}
}

8
lucene/contrib/ant/src/test/org/apache/lucene/ant/TextDocumentTest.java
View File

@ -17,8 +17,6 @@ package org.apache.lucene.ant;
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.ant.DocumentTestCase;
import org.apache.lucene.ant.TextDocument;
@ -27,7 +25,8 @@ public class TextDocumentTest extends DocumentTestCase
TextDocument doc;
@Override
public void setUp() throws IOException {
public void setUp() throws Exception {
super.setUp();
doc = new TextDocument(getFile("test.txt"));
}
@ -36,8 +35,9 @@ public class TextDocumentTest extends DocumentTestCase
}
@Override
public void tearDown() {
public void tearDown() throws Exception {
doc = null;
super.tearDown();
}
}

2
lucene/contrib/db/bdb-je/lib/je-3.3.93.jar Normal file
View File

@ -0,0 +1,2 @@
AnyObjectId[9a9ff077cdd36a96e7e0506986edd4e52b90a22f] was removed in git history.
Apache SVN contains full history.

1
lucene/contrib/db/bdb-je/lib/je-LICENSE-FAKE.txt Normal file
View File

@ -0,0 +1 @@
No bdb jars are shipped with lucene. This is a fake license to work around the automated license checking.

1
lucene/contrib/db/bdb-je/lib/je-NOTICE-FAKE.txt Normal file
View File

@ -0,0 +1 @@
No bdb jars are shipped with lucene. This is a fake license to work around the automated license checking.

11
lucene/contrib/db/bdb-je/src/java/org/apache/lucene/store/je/JEDirectory.java
View File

@ -199,17 +199,6 @@ public class JEDirectory extends Directory {
return new JELock();
}
@Override
public void touchFile(String name) throws IOException {
File file = new File(name);
long length = 0L;
if (file.exists(this))
length = file.getLength();
file.modify(this, length, System.currentTimeMillis());
}
/**
* Once a transaction handle was committed it is no longer valid. In order
* to continue using this JEDirectory instance after a commit, the

1
lucene/contrib/db/bdb/lib/db--NOTICE-FAKE.txt Normal file
View File

@ -0,0 +1 @@
No bdb jars are shipped with lucene. This is a fake license to work around the automated license checking.

2
lucene/contrib/db/bdb/lib/db-4.7.25.jar Normal file
View File

@ -0,0 +1,2 @@
AnyObjectId[99baf20bacd712cae91dd6e4e1f46224cafa1a37] was removed in git history.
Apache SVN contains full history.

1
lucene/contrib/db/bdb/lib/db-LICENSE-FAKE.txt Normal file
View File

@ -0,0 +1 @@
No bdb jars are shipped with lucene. This is a fake license to work around the automated license checking.

13
lucene/contrib/db/bdb/src/java/org/apache/lucene/store/db/DbDirectory.java
View File

@ -222,19 +222,6 @@ public class DbDirectory extends Directory {
return new DbLock();
}
@Override
public void touchFile(String name)
throws IOException
{
File file = new File(name);
long length = 0L;
if (file.exists(this))
length = file.getLength();
file.modify(this, length, System.currentTimeMillis());
}
/**
* Once a transaction handle was committed it is no longer valid. In
* order to continue using this DbDirectory instance after a commit, the

20
lucene/contrib/demo/src/test/org/apache/lucene/demo/TestDemo.java
View File

@ -22,16 +22,17 @@ import java.io.File;
import java.io.PrintStream;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
public class TestDemo extends LuceneTestCase {
private void testOneSearch(String query, int expectedHitCount) throws Exception {
private void testOneSearch(File indexPath, String query, int expectedHitCount) throws Exception {
PrintStream outSave = System.out;
try {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
PrintStream fakeSystemOut = new PrintStream(bytes);
System.setOut(fakeSystemOut);
SearchFiles.main(new String[] {"-query", query});
SearchFiles.main(new String[] {"-query", query, "-index", indexPath.getPath()});
fakeSystemOut.flush();
String output = bytes.toString(); // intentionally use default encoding
assertTrue("output=" + output, output.contains(expectedHitCount + " total matching documents"));
@ -42,12 +43,13 @@ public class TestDemo extends LuceneTestCase {
public void testIndexSearch() throws Exception {
File dir = getDataFile("test-files/docs");
IndexFiles.main(new String[] { "-create", "-docs", dir.getPath() });
testOneSearch("apache", 3);
testOneSearch("patent", 8);
testOneSearch("lucene", 0);
testOneSearch("gnu", 6);
testOneSearch("derivative", 8);
testOneSearch("license", 13);
File indexDir = _TestUtil.getTempDir("ContribDemoTest");
IndexFiles.main(new String[] { "-create", "-docs", dir.getPath(), "-index", indexDir.getPath()});
testOneSearch(indexDir, "apache", 3);
testOneSearch(indexDir, "patent", 8);
testOneSearch(indexDir, "lucene", 0);
testOneSearch(indexDir, "gnu", 6);
testOneSearch(indexDir, "derivative", 8);
testOneSearch(indexDir, "license", 13);
}
}

10
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/Highlighter.java
View File

@ -78,7 +78,7 @@ public class Highlighter
public final String getBestFragment(Analyzer analyzer, String fieldName,String text)
throws IOException, InvalidTokenOffsetsException
{
TokenStream tokenStream = analyzer.tokenStream(fieldName, new StringReader(text));
TokenStream tokenStream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
return getBestFragment(tokenStream, text);
}
@ -130,7 +130,7 @@ public class Highlighter
int maxNumFragments)
throws IOException, InvalidTokenOffsetsException
{
TokenStream tokenStream = analyzer.tokenStream(fieldName, new StringReader(text));
TokenStream tokenStream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
return getBestFragments(tokenStream, text, maxNumFragments);
}
@ -197,6 +197,11 @@ public class Highlighter
tokenStream.reset();
TextFragment currentFrag = new TextFragment(newText,newText.length(), docFrags.size());
if (fragmentScorer instanceof QueryScorer) {
((QueryScorer) fragmentScorer).setMaxDocCharsToAnalyze(maxDocCharsToAnalyze);
}
TokenStream newStream = fragmentScorer.init(tokenStream);
if(newStream != null) {
tokenStream = newStream;
@ -350,6 +355,7 @@ public class Highlighter
{
try
{
tokenStream.end();
tokenStream.close();
}
catch (Exception e)

57
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/OffsetLimitTokenFilter.java Normal file
View File

@ -0,0 +1,57 @@
package org.apache.lucene.search.highlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
/**
* This TokenFilter limits the number of tokens while indexing by adding up the
* current offset.
*/
public final class OffsetLimitTokenFilter extends TokenFilter {
private int offsetCount;
private OffsetAttribute offsetAttrib = getAttribute(OffsetAttribute.class);
private int offsetLimit;
public OffsetLimitTokenFilter(TokenStream input, int offsetLimit) {
super(input);
this.offsetLimit = offsetLimit;
}
@Override
public boolean incrementToken() throws IOException {
if (offsetCount < offsetLimit && input.incrementToken()) {
int offsetLength = offsetAttrib.endOffset() - offsetAttrib.startOffset();
offsetCount += offsetLength;
return true;
}
return false;
}
@Override
public void reset() throws IOException {
super.reset();
offsetCount = 0;
}
}

7
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/QueryScorer.java
View File

@ -54,6 +54,7 @@ public class QueryScorer implements Scorer {
private IndexReader reader;
private boolean skipInitExtractor;
private boolean wrapToCaching = true;
private int maxCharsToAnalyze;
/**
* @param query Query to use for highlighting
@ -209,7 +210,7 @@ public class QueryScorer implements Scorer {
private TokenStream initExtractor(TokenStream tokenStream) throws IOException {
WeightedSpanTermExtractor qse = defaultField == null ? new WeightedSpanTermExtractor()
: new WeightedSpanTermExtractor(defaultField);
qse.setMaxDocCharsToAnalyze(maxCharsToAnalyze);
qse.setExpandMultiTermQuery(expandMultiTermQuery);
qse.setWrapIfNotCachingTokenFilter(wrapToCaching);
if (reader == null) {
@ -265,4 +266,8 @@ public class QueryScorer implements Scorer {
public void setWrapIfNotCachingTokenFilter(boolean wrap) {
this.wrapToCaching = wrap;
}
public void setMaxDocCharsToAnalyze(int maxDocCharsToAnalyze) {
this.maxCharsToAnalyze = maxDocCharsToAnalyze;
}
}

35
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/TokenSources.java
View File

@ -30,6 +30,7 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.TermFreqVector;
@ -158,10 +159,13 @@ public class TokenSources {
OffsetAttribute offsetAtt;
PositionIncrementAttribute posincAtt;
StoredTokenStream(Token tokens[]) {
this.tokens = tokens;
termAtt = addAttribute(CharTermAttribute.class);
offsetAtt = addAttribute(OffsetAttribute.class);
posincAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
}
@Override
@ -173,6 +177,10 @@ public class TokenSources {
clearAttributes();
termAtt.setEmpty().append(token);
offsetAtt.setOffset(token.startOffset(), token.endOffset());
posincAtt
.setPositionIncrement(currentToken <= 1
|| tokens[currentToken - 1].startOffset() > tokens[currentToken - 2]
.startOffset() ? 1 : 0);
return true;
}
}
@ -180,7 +188,6 @@ public class TokenSources {
BytesRef[] terms = tpv.getTerms();
int[] freq = tpv.getTermFrequencies();
int totalTokens = 0;
for (int t = 0; t < freq.length; t++) {
totalTokens += freq[t];
}
@ -189,7 +196,8 @@ public class TokenSources {
for (int t = 0; t < freq.length; t++) {
TermVectorOffsetInfo[] offsets = tpv.getOffsets(t);
if (offsets == null) {
throw new IllegalArgumentException("Required TermVector Offset information was not found");
throw new IllegalArgumentException(
"Required TermVector Offset information was not found");
}
int[] pos = null;
@ -205,8 +213,8 @@ public class TokenSources {
unsortedTokens = new ArrayList<Token>();
}
for (int tp = 0; tp < offsets.length; tp++) {
Token token = new Token(terms[t].utf8ToString(), offsets[tp].getStartOffset(), offsets[tp]
.getEndOffset());
Token token = new Token(terms[t].utf8ToString(),
offsets[tp].getStartOffset(), offsets[tp].getEndOffset());
unsortedTokens.add(token);
}
} else {
@ -221,8 +229,8 @@ public class TokenSources {
// tokens stored with positions - can use this to index straight into
// sorted array
for (int tp = 0; tp < pos.length; tp++) {
Token token = new Token(terms[t].utf8ToString(), offsets[tp].getStartOffset(),
offsets[tp].getEndOffset());
Token token = new Token(terms[t].utf8ToString(),
offsets[tp].getStartOffset(), offsets[tp].getEndOffset());
tokensInOriginalOrder[pos[tp]] = token;
}
}
@ -231,12 +239,11 @@ public class TokenSources {
if (unsortedTokens != null) {
tokensInOriginalOrder = unsortedTokens.toArray(new Token[unsortedTokens
.size()]);
ArrayUtil.quickSort(tokensInOriginalOrder, new Comparator<Token>() {
ArrayUtil.mergeSort(tokensInOriginalOrder, new Comparator<Token>() {
public int compare(Token t1, Token t2) {
if (t1.startOffset() == t2.startOffset())
return t1.endOffset() - t2.endOffset();
else
return t1.startOffset() - t2.startOffset();
if (t1.startOffset() == t2.startOffset()) return t1.endOffset()
- t2.endOffset();
else return t1.startOffset() - t2.startOffset();
}
});
}
@ -279,7 +286,11 @@ public class TokenSources {
// convenience method
public static TokenStream getTokenStream(String field, String contents,
Analyzer analyzer) {
return analyzer.tokenStream(field, new StringReader(contents));
try {
return analyzer.reusableTokenStream(field, new StringReader(contents));
} catch (IOException ex) {
throw new RuntimeException(ex);
}
}
}

9
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
View File

@ -56,6 +56,7 @@ public class WeightedSpanTermExtractor {
private boolean expandMultiTermQuery;
private boolean cachedTokenStream;
private boolean wrapToCaching = true;
private int maxDocCharsToAnalyze;
public WeightedSpanTermExtractor() {
}
@ -320,13 +321,13 @@ public class WeightedSpanTermExtractor {
private AtomicReaderContext getLeafContextForField(String field) throws IOException {
if(wrapToCaching && !cachedTokenStream && !(tokenStream instanceof CachingTokenFilter)) {
tokenStream = new CachingTokenFilter(tokenStream);
tokenStream = new CachingTokenFilter(new OffsetLimitTokenFilter(tokenStream, maxDocCharsToAnalyze));
cachedTokenStream = true;
}
AtomicReaderContext context = readers.get(field);
if (context == null) {
MemoryIndex indexer = new MemoryIndex();
indexer.addField(field, tokenStream);
indexer.addField(field, new OffsetLimitTokenFilter(tokenStream, maxDocCharsToAnalyze));
tokenStream.reset();
IndexSearcher searcher = indexer.createSearcher();
// MEM index has only atomic ctx
@ -545,4 +546,8 @@ public class WeightedSpanTermExtractor {
public void setWrapIfNotCachingTokenFilter(boolean wrap) {
this.wrapToCaching = wrap;
}
protected final void setMaxDocCharsToAnalyze(int maxDocCharsToAnalyze) {
this.maxDocCharsToAnalyze = maxDocCharsToAnalyze;
}
}

10
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
View File

@ -58,7 +58,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
final String TEXT = "the fox jumped";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamConcurrent(),
@ -102,7 +102,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
final String TEXT = "the fox jumped";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamConcurrent(),
@ -172,7 +172,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamSparse(),
@ -215,7 +215,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, TEXT, Store.YES, Index.ANALYZED,
@ -256,7 +256,7 @@ public class HighlighterPhraseTest extends LuceneTestCase {
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamSparse(),

68
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
View File

@ -90,7 +90,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
Directory ramDir;
public IndexSearcher searcher = null;
int numHighlights = 0;
final Analyzer analyzer = new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
final Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
TopDocs hits;
String[] texts = {
@ -101,7 +101,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
"wordx wordy wordz wordx wordy wordx worda wordb wordy wordc", "y z x y z a b", "lets is a the lets is a the lets is a the lets" };
public void testQueryScorerHits() throws Exception {
Analyzer analyzer = new MockAnalyzer(MockTokenizer.SIMPLE, true);
Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, analyzer);
query = qp.parse("\"very long\"");
searcher = new IndexSearcher(ramDir, true);
@ -133,7 +133,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
String s1 = "I call our world Flatland, not because we call it so,";
QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true));
// Verify that a query against the default field results in text being
// highlighted
@ -165,7 +165,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
*/
private static String highlightField(Query query, String fieldName, String text)
throws IOException, InvalidTokenOffsetsException {
TokenStream tokenStream = new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true).tokenStream(fieldName, new StringReader(text));
TokenStream tokenStream = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true).tokenStream(fieldName, new StringReader(text));
// Assuming "<B>", "</B>" used to highlight
SimpleHTMLFormatter formatter = new SimpleHTMLFormatter();
QueryScorer scorer = new QueryScorer(query, fieldName, FIELD_NAME);
@ -210,7 +210,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
String f2c = f2 + ":";
String q = "(" + f1c + ph1 + " OR " + f2c + ph1 + ") AND (" + f1c + ph2
+ " OR " + f2c + ph2 + ")";
Analyzer analyzer = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, f1, analyzer);
Query query = qp.parse(q);
@ -1093,6 +1093,10 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
}
public void testMaxSizeHighlight() throws Exception {
final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
// we disable MockTokenizer checks because we will forcefully limit the
// tokenstream and call end() before incrementToken() returns false.
analyzer.setEnableChecks(false);
TestHighlightRunner helper = new TestHighlightRunner() {
@Override
@ -1122,7 +1126,10 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
public void run() throws Exception {
String goodWord = "goodtoken";
CharacterRunAutomaton stopWords = new CharacterRunAutomaton(BasicAutomata.makeString("stoppedtoken"));
// we disable MockTokenizer checks because we will forcefully limit the
// tokenstream and call end() before incrementToken() returns false.
final MockAnalyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true);
analyzer.setEnableChecks(false);
TermQuery query = new TermQuery(new Term("data", goodWord));
String match;
@ -1134,13 +1141,13 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
sb.append("stoppedtoken");
}
SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
Highlighter hg = getHighlighter(query, "data", new MockAnalyzer(MockTokenizer.SIMPLE, true, stopWords, true).tokenStream(
Highlighter hg = getHighlighter(query, "data", analyzer.tokenStream(
"data", new StringReader(sb.toString())), fm);// new Highlighter(fm,
// new
// QueryTermScorer(query));
hg.setTextFragmenter(new NullFragmenter());
hg.setMaxDocCharsToAnalyze(100);
match = hg.getBestFragment(new MockAnalyzer(MockTokenizer.SIMPLE, true, stopWords, true), "data", sb.toString());
match = hg.getBestFragment(analyzer, "data", sb.toString());
assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg
.getMaxDocCharsToAnalyze());
@ -1151,7 +1158,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
// + whitespace)
sb.append(" ");
sb.append(goodWord);
match = hg.getBestFragment(new MockAnalyzer(MockTokenizer.SIMPLE, true, stopWords, true), "data", sb.toString());
match = hg.getBestFragment(analyzer, "data", sb.toString());
assertTrue("Matched text should be no more than 100 chars in length ", match.length() < hg
.getMaxDocCharsToAnalyze());
}
@ -1170,10 +1177,10 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
String text = "this is a text with searchterm in it";
SimpleHTMLFormatter fm = new SimpleHTMLFormatter();
Highlighter hg = getHighlighter(query, "text", new MockAnalyzer(MockTokenizer.SIMPLE, true, stopWords, true).tokenStream("text", new StringReader(text)), fm);
Highlighter hg = getHighlighter(query, "text", new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true).tokenStream("text", new StringReader(text)), fm);
hg.setTextFragmenter(new NullFragmenter());
hg.setMaxDocCharsToAnalyze(36);
String match = hg.getBestFragment(new MockAnalyzer(MockTokenizer.SIMPLE, true, stopWords, true), "text", text);
String match = hg.getBestFragment(new MockAnalyzer(random, MockTokenizer.SIMPLE, true, stopWords, true), "text", text);
assertTrue(
"Matched text should contain remainder of text after highlighted query ",
match.endsWith("in it"));
@ -1191,7 +1198,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
// test to show how rewritten query can still be used
if (searcher != null) searcher.close();
searcher = new IndexSearcher(ramDir, true);
Analyzer analyzer = new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
Analyzer analyzer = new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
QueryParser parser = new QueryParser(TEST_VERSION_CURRENT, FIELD_NAME, analyzer);
Query query = parser.parse("JF? or Kenned*");
@ -1446,64 +1453,64 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
Highlighter highlighter;
String result;
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("foo");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("foo");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-Speed10 <B>foo</B>", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("10");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("10");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-Speed<B>10</B> foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("hi");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi</B>-Speed10 foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("speed");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("speed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("Hi-<B>Speed</B>10 foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("hispeed");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hispeed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("hi speed");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi speed");
highlighter = getHighlighter(query, "text", getTS2(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
// ///////////////// same tests, just put the bigger overlapping token
// first
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("foo");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("foo");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-Speed10 <B>foo</B>", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("10");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("10");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-Speed<B>10</B> foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("hi");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi</B>-Speed10 foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("speed");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("speed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("Hi-<B>Speed</B>10 foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("hispeed");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hispeed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(MockTokenizer.WHITESPACE, false)).parse("hi speed");
query = new QueryParser(TEST_VERSION_CURRENT, "text", new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).parse("hi speed");
highlighter = getHighlighter(query, "text", getTS2a(), HighlighterTest.this);
result = highlighter.getBestFragments(getTS2a(), s, 3, "...");
assertEquals("<B>Hi-Speed</B>10 foo", result);
@ -1514,7 +1521,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
}
private Directory dir;
private Analyzer a = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
private Analyzer a = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
public void testWeightedTermsWithDeletes() throws IOException, ParseException, InvalidTokenOffsetsException {
makeIndex();
@ -1529,7 +1536,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
}
private void makeIndex() throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)));
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)));
writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );
writer.addDocument( doc( "t_text1", "more random words for second field del" ) );
writer.addDocument( doc( "t_text1", "random words for highlighting tests del" ) );
@ -1539,7 +1546,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
}
private void deleteDocument() throws IOException {
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false)).setOpenMode(OpenMode.APPEND));
IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false)).setOpenMode(OpenMode.APPEND));
writer.deleteDocuments( new Term( "t_text1", "del" ) );
// To see negative idf, keep comment the following line
//writer.optimize();
@ -1644,7 +1651,7 @@ public class HighlighterTest extends BaseTokenStreamTestCase implements Formatte
dir = newDirectory();
ramDir = newDirectory();
IndexWriter writer = new IndexWriter(ramDir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true)));
for (String text : texts) {
addDoc(writer, text);
}
@ -1726,6 +1733,11 @@ final class SynonymAnalyzer extends Analyzer {
stream.addAttribute(CharTermAttribute.class);
stream.addAttribute(PositionIncrementAttribute.class);
stream.addAttribute(OffsetAttribute.class);
try {
stream.reset();
} catch (IOException e) {
throw new RuntimeException(e);
}
return new SynonymTokenizer(stream, synonyms);
}
}

66
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/OffsetLimitTokenFilterTest.java Normal file
View File

@ -0,0 +1,66 @@
package org.apache.lucene.search.highlight;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.Reader;
import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.TokenStream;
public class OffsetLimitTokenFilterTest extends BaseTokenStreamTestCase {
public void testFilter() throws Exception {
// we disable MockTokenizer checks because we will forcefully limit the
// tokenstream and call end() before incrementToken() returns false.
MockTokenizer stream = new MockTokenizer(new StringReader(
"short toolong evenmuchlongertext a ab toolong foo"),
MockTokenizer.WHITESPACE, false);
stream.setEnableChecks(false);
OffsetLimitTokenFilter filter = new OffsetLimitTokenFilter(stream, 10);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
stream = new MockTokenizer(new StringReader(
"short toolong evenmuchlongertext a ab toolong foo"),
MockTokenizer.WHITESPACE, false);
stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 12);
assertTokenStreamContents(filter, new String[] {"short", "toolong"});
stream = new MockTokenizer(new StringReader(
"short toolong evenmuchlongertext a ab toolong foo"),
MockTokenizer.WHITESPACE, false);
stream.setEnableChecks(false);
filter = new OffsetLimitTokenFilter(stream, 30);
assertTokenStreamContents(filter, new String[] {"short", "toolong",
"evenmuchlongertext"});
// TODO: This is not actually testing reuse! (reusableTokenStream is not implemented)
checkOneTermReuse(new Analyzer() {
@Override
public TokenStream tokenStream(String fieldName, Reader reader) {
MockTokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.WHITESPACE, false);
tokenizer.setEnableChecks(false);
return new OffsetLimitTokenFilter(tokenizer, 10);
}
}, "llenges", "llenges");
}
}

108
lucene/contrib/highlighter/src/test/org/apache/lucene/search/highlight/TokenSourcesTest.java
View File

@ -36,7 +36,10 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermPositionVector;
import org.apache.lucene.search.DisjunctionMaxQuery;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.LockObtainFailedException;
@ -86,12 +89,12 @@ public class TokenSourcesTest extends LuceneTestCase {
public void reset() {
this.i = -1;
this.tokens = new Token[] {
new Token(new char[] { 't', 'h', 'e' }, 0, 3, 0, 3),
new Token(new char[] { '{', 'f', 'o', 'x', '}' }, 0, 5, 0, 7),
new Token(new char[] { 'f', 'o', 'x' }, 0, 3, 4, 7),
new Token(new char[] { 'd', 'i', 'd' }, 0, 3, 8, 11),
new Token(new char[] { 'n', 'o', 't' }, 0, 3, 12, 15),
new Token(new char[] { 'j', 'u', 'm', 'p' }, 0, 4, 16, 20) };
new Token(new char[] {'t', 'h', 'e'}, 0, 3, 0, 3),
new Token(new char[] {'{', 'f', 'o', 'x', '}'}, 0, 5, 0, 7),
new Token(new char[] {'f', 'o', 'x'}, 0, 3, 4, 7),
new Token(new char[] {'d', 'i', 'd'}, 0, 3, 8, 11),
new Token(new char[] {'n', 'o', 't'}, 0, 3, 12, 15),
new Token(new char[] {'j', 'u', 'm', 'p'}, 0, 4, 16, 20)};
this.tokens[1].setPositionIncrement(0);
}
}
@ -188,4 +191,97 @@ public class TokenSourcesTest extends LuceneTestCase {
}
}
public void testOverlapWithOffsetExactPhrase() throws CorruptIndexException,
LockObtainFailedException, IOException, InvalidTokenOffsetsException {
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamOverlap(),
TermVector.WITH_OFFSETS));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
}
final IndexReader indexReader = IndexReader.open(directory, true);
try {
assertEquals(1, indexReader.numDocs());
final IndexSearcher indexSearcher = newSearcher(indexReader);
try {
// final DisjunctionMaxQuery query = new DisjunctionMaxQuery(1);
// query.add(new SpanTermQuery(new Term(FIELD, "{fox}")));
// query.add(new SpanTermQuery(new Term(FIELD, "fox")));
final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {
new SpanTermQuery(new Term(FIELD, "the")),
new SpanTermQuery(new Term(FIELD, "fox"))}, 0, true);
TopDocs hits = indexSearcher.search(phraseQuery, 1);
assertEquals(1, hits.totalHits);
final Highlighter highlighter = new Highlighter(
new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
new QueryScorer(phraseQuery));
final TokenStream tokenStream = TokenSources
.getTokenStream(
(TermPositionVector) indexReader.getTermFreqVector(0, FIELD),
false);
assertEquals("<B>the fox</B> did not jump",
highlighter.getBestFragment(tokenStream, TEXT));
} finally {
indexSearcher.close();
}
} finally {
indexReader.close();
directory.close();
}
}
public void testOverlapWithPositionsAndOffsetExactPhrase()
throws CorruptIndexException, LockObtainFailedException, IOException,
InvalidTokenOffsetsException {
final String TEXT = "the fox did not jump";
final Directory directory = newDirectory();
final IndexWriter indexWriter = new IndexWriter(directory,
newIndexWriterConfig(TEST_VERSION_CURRENT, new OverlapAnalyzer()));
try {
final Document document = new Document();
document.add(new Field(FIELD, new TokenStreamOverlap(),
TermVector.WITH_POSITIONS_OFFSETS));
indexWriter.addDocument(document);
} finally {
indexWriter.close();
}
final IndexReader indexReader = IndexReader.open(directory, true);
try {
assertEquals(1, indexReader.numDocs());
final IndexSearcher indexSearcher = newSearcher(indexReader);
try {
// final DisjunctionMaxQuery query = new DisjunctionMaxQuery(1);
// query.add(new SpanTermQuery(new Term(FIELD, "the")));
// query.add(new SpanTermQuery(new Term(FIELD, "fox")));
final Query phraseQuery = new SpanNearQuery(new SpanQuery[] {
new SpanTermQuery(new Term(FIELD, "the")),
new SpanTermQuery(new Term(FIELD, "fox"))}, 0, true);
TopDocs hits = indexSearcher.search(phraseQuery, 1);
assertEquals(1, hits.totalHits);
final Highlighter highlighter = new Highlighter(
new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
new QueryScorer(phraseQuery));
final TokenStream tokenStream = TokenSources
.getTokenStream(
(TermPositionVector) indexReader.getTermFreqVector(0, FIELD),
false);
assertEquals("<B>the fox</B> did not jump",
highlighter.getBestFragment(tokenStream, TEXT));
} finally {
indexSearcher.close();
}
} finally {
indexReader.close();
directory.close();
}
}
}

4
lucene/contrib/highlighter/src/test/org/apache/lucene/search/vectorhighlight/AbstractTestCase.java
View File

@ -87,9 +87,9 @@ public abstract class AbstractTestCase extends LuceneTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
analyzerW = new MockAnalyzer(MockTokenizer.WHITESPACE, false);
analyzerW = new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
analyzerB = new BigramAnalyzer();
analyzerK = new MockAnalyzer(MockTokenizer.KEYWORD, false);
analyzerK = new MockAnalyzer(random, MockTokenizer.KEYWORD, false);
paW = new QueryParser(TEST_VERSION_CURRENT, F, analyzerW );
paB = new QueryParser(TEST_VERSION_CURRENT, F, analyzerB );
dir = newDirectory();

2
lucene/contrib/instantiated/src/java/org/apache/lucene/store/instantiated/InstantiatedIndexWriter.java
View File

@ -532,7 +532,7 @@ public class InstantiatedIndexWriter implements Closeable {
if (field.tokenStreamValue() != null) {
tokenStream = field.tokenStreamValue();
} else {
tokenStream = analyzer.tokenStream(field.name(), new StringReader(field.stringValue()));
tokenStream = analyzer.reusableTokenStream(field.name(), new StringReader(field.stringValue()));
}
// reset the TokenStream to the first token

4
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestEmptyIndex.java
View File

@ -59,7 +59,7 @@ public class TestEmptyIndex extends LuceneTestCase {
// make sure a Directory acts the same
Directory d = newDirectory();
new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())).close();
new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))).close();
r = IndexReader.open(d, false);
testNorms(r);
r.close();
@ -84,7 +84,7 @@ public class TestEmptyIndex extends LuceneTestCase {
// make sure a Directory acts the same
Directory d = newDirectory();
new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer())).close();
new IndexWriter(d, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random))).close();
r = IndexReader.open(d, false);
termsEnumTest(r);
r.close();

12
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestIndicesEquals.java
View File

@ -21,6 +21,7 @@ import java.util.Arrays;
import java.util.Comparator;
import java.util.Iterator;
import java.util.List;
import java.util.Random;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
@ -65,7 +66,7 @@ public class TestIndicesEquals extends LuceneTestCase {
// create dir data
IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < 20; i++) {
Document document = new Document();
@ -88,10 +89,13 @@ public class TestIndicesEquals extends LuceneTestCase {
Directory dir = newDirectory();
InstantiatedIndex ii = new InstantiatedIndex();
// we need to pass the "same" random to both, so they surely index the same payload data.
long seed = random.nextLong();
// create dir data
IndexWriter indexWriter = new IndexWriter(dir, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
TEST_VERSION_CURRENT, new MockAnalyzer(new Random(seed))).setMergePolicy(newLogMergePolicy()));
indexWriter.setInfoStream(VERBOSE ? System.out : null);
if (VERBOSE) {
System.out.println("TEST: make test index");
@ -104,7 +108,7 @@ public class TestIndicesEquals extends LuceneTestCase {
indexWriter.close();
// test ii writer
InstantiatedIndexWriter instantiatedIndexWriter = ii.indexWriterFactory(new MockAnalyzer(), true);
InstantiatedIndexWriter instantiatedIndexWriter = ii.indexWriterFactory(new MockAnalyzer(new Random(seed)), true);
for (int i = 0; i < 500; i++) {
Document document = new Document();
assembleDocument(document, i);

2
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestRealTime.java
View File

@ -36,7 +36,7 @@ public class TestRealTime extends LuceneTestCase {
InstantiatedIndex index = new InstantiatedIndex();
InstantiatedIndexReader reader = new InstantiatedIndexReader(index);
IndexSearcher searcher = newSearcher(reader);
IndexSearcher searcher = newSearcher(reader, false);
InstantiatedIndexWriter writer = new InstantiatedIndexWriter(index);
Document doc;

6
lucene/contrib/instantiated/src/test/org/apache/lucene/store/instantiated/TestUnoptimizedReaderOnConstructor.java
View File

@ -34,17 +34,17 @@ public class TestUnoptimizedReaderOnConstructor extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
addDocument(iw, "Hello, world!");
addDocument(iw, "All work and no play makes jack a dull boy");
iw.close();
iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
addDocument(iw, "Hello, tellus!");
addDocument(iw, "All work and no play makes danny a dull boy");
iw.close();
iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setOpenMode(OpenMode.APPEND));
iw = new IndexWriter(dir, new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setOpenMode(OpenMode.APPEND));
addDocument(iw, "Hello, earth!");
addDocument(iw, "All work and no play makes wendy a dull girl");
iw.close();

4
lucene/contrib/lucli/src/java/lucli/LuceneMethods.java
View File

@ -305,11 +305,12 @@ class LuceneMethods {
int position = 0;
// Tokenize field and add to postingTable
TokenStream stream = analyzer.tokenStream(fieldName, reader);
TokenStream stream = analyzer.reusableTokenStream(fieldName, reader);
CharTermAttribute termAtt = stream.addAttribute(CharTermAttribute.class);
PositionIncrementAttribute posIncrAtt = stream.addAttribute(PositionIncrementAttribute.class);
try {
stream.reset();
while (stream.incrementToken()) {
position += (posIncrAtt.getPositionIncrement() - 1);
position++;
@ -323,6 +324,7 @@ class LuceneMethods {
}
if (position > maxFieldLength) break;
}
stream.end();
} finally {
stream.close();
}

8
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
View File

@ -261,8 +261,12 @@ public class MemoryIndex {
if (analyzer == null)
throw new IllegalArgumentException("analyzer must not be null");
TokenStream stream = analyzer.tokenStream(fieldName,
new StringReader(text));
TokenStream stream;
try {
stream = analyzer.reusableTokenStream(fieldName, new StringReader(text));
} catch (IOException ex) {
throw new RuntimeException(ex);
}
addField(fieldName, stream);
}

6
lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
View File

@ -143,9 +143,9 @@ public class MemoryIndexTest extends BaseTokenStreamTestCase {
*/
private Analyzer randomAnalyzer() {
switch(random.nextInt(3)) {
case 0: return new MockAnalyzer(MockTokenizer.SIMPLE, true);
case 1: return new MockAnalyzer(MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
default: return new MockAnalyzer(MockTokenizer.WHITESPACE, false);
case 0: return new MockAnalyzer(random, MockTokenizer.SIMPLE, true);
case 1: return new MockAnalyzer(random, MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET, true);
default: return new MockAnalyzer(random, MockTokenizer.WHITESPACE, false);
}
}

26
lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.util.Collections;
import java.util.Set;
/**
@ -135,7 +136,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
if (last > 1 || !isOptimized(infos.info(0))) {
spec = new MergeSpecification();
spec.add(new OneMerge(infos.range(0, last)));
spec.add(new OneMerge(infos.asList().subList(0, last)));
}
} else if (last > maxNumSegments) {
@ -192,7 +193,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
prev = backLink[i][prev];
int mergeStart = i + prev;
if((mergeEnd - mergeStart) > 1) {
spec.add(new OneMerge(infos.range(mergeStart, mergeEnd)));
spec.add(new OneMerge(infos.asList().subList(mergeStart, mergeEnd)));
} else {
if(partialExpunge) {
SegmentInfo info = infos.info(mergeStart);
@ -208,7 +209,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
if(partialExpunge && maxDelCount > 0) {
// expunge deletes
spec.add(new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1)));
spec.add(new OneMerge(Collections.singletonList(infos.info(expungeCandidate))));
}
return spec;
@ -250,7 +251,10 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
MergeSpecification spec = null;
if(numLargeSegs < numSegs) {
SegmentInfos smallSegments = infos.range(numLargeSegs, numSegs);
// hack to create a shallow sub-range as SegmentInfos instance,
// it does not clone all metadata, but LogMerge does not need it
final SegmentInfos smallSegments = new SegmentInfos();
smallSegments.rollbackSegmentInfos(infos.asList().subList(numLargeSegs, numSegs));
spec = super.findMergesToExpungeDeletes(smallSegments);
}
@ -258,7 +262,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
for(int i = 0; i < numLargeSegs; i++) {
SegmentInfo info = infos.info(i);
if(info.hasDeletions()) {
spec.add(new OneMerge(infos.range(i, i + 1)));
spec.add(new OneMerge(Collections.singletonList(infos.info(i))));
}
}
return spec;
@ -296,7 +300,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
if(totalSmallSegSize < targetSegSize * 2) {
MergeSpecification spec = findBalancedMerges(infos, numLargeSegs, (numLargeSegs - 1), _partialExpunge);
if(spec == null) spec = new MergeSpecification(); // should not happen
spec.add(new OneMerge(infos.range(numLargeSegs, numSegs)));
spec.add(new OneMerge(infos.asList().subList(numLargeSegs, numSegs)));
return spec;
} else {
return findBalancedMerges(infos, numSegs, numLargeSegs, _partialExpunge);
@ -311,11 +315,13 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
if(size(info) < sizeThreshold) break;
startSeg++;
}
spec.add(new OneMerge(infos.range(startSeg, numSegs)));
spec.add(new OneMerge(infos.asList().subList(startSeg, numSegs)));
return spec;
} else {
// apply the log merge policy to small segments.
SegmentInfos smallSegments = infos.range(numLargeSegs, numSegs);
// hack to create a shallow sub-range as SegmentInfos instance,
// it does not clone all metadata, but LogMerge does not need it
final SegmentInfos smallSegments = new SegmentInfos();
smallSegments.rollbackSegmentInfos(infos.asList().subList(numLargeSegs, numSegs));
MergeSpecification spec = super.findMerges(smallSegments);
if(_partialExpunge) {
@ -342,7 +348,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
}
}
if (maxDelCount > 0) {
return new OneMerge(infos.range(expungeCandidate, expungeCandidate + 1));
return new OneMerge(Collections.singletonList(infos.info(expungeCandidate)));
}
return null;
}

280
lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java Normal file
View File

@ -0,0 +1,280 @@
package org.apache.lucene.store;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.IOException;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.lucene.index.ConcurrentMergeScheduler;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.index.IndexWriter; // javadocs
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergeScheduler;
import org.apache.lucene.store.RAMDirectory; // javadocs
import org.apache.lucene.util.IOUtils;
// TODO
// - let subclass dictate policy...?
// - rename to MergeCacheingDir? NRTCachingDir
/**
* Wraps a {@link RAMDirectory}
* around any provided delegate directory, to
* be used during NRT search. Make sure you pull the merge
* scheduler using {@link #getMergeScheduler} and pass that to your
* {@link IndexWriter}; this class uses that to keep track of which
* merges are being done by which threads, to decide when to
* cache each written file.
*
* <p>This class is likely only useful in a near-real-time
* context, where indexing rate is lowish but reopen
* rate is highish, resulting in many tiny files being
* written. This directory keeps such segments (as well as
* the segments produced by merging them, as long as they
* are small enough), in RAM.</p>
*
* <p>This is safe to use: when your app calls {IndexWriter#commit},
* all cached files will be flushed from the cached and sync'd.</p>
*
* <p><b>NOTE</b>: this class is somewhat sneaky in its
* approach for spying on merges to determine the size of a
* merge: it records which threads are running which merges
* by watching ConcurrentMergeScheduler's doMerge method.
* While this works correctly, likely future versions of
* this class will take a more general approach.
*
* <p>Here's a simple example usage:
*
* <pre>
* Directory fsDir = FSDirectory.open(new File("/path/to/index"));
* NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 5.0, 60.0);
* IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_32, analyzer);
* conf.setMergeScheduler(cachedFSDir.getMergeScheduler());
* IndexWriter writer = new IndexWriter(cachedFSDir, conf);
* </pre>
*
* <p>This will cache all newly flushed segments, all merges
* whose expected segment size is <= 5 MB, unless the net
* cached bytes exceeds 60 MB at which point all writes will
* not be cached (until the net bytes falls below 60 MB).</p>
*
* @lucene.experimental
*/
public class NRTCachingDirectory extends Directory {
private final RAMDirectory cache = new RAMDirectory();
private final Directory delegate;
private final long maxMergeSizeBytes;
private final long maxCachedBytes;
private static final boolean VERBOSE = false;
/**
* We will cache a newly created output if 1) it's a
* flush or a merge and the estimated size of the merged segmnt is <=
* maxMergeSizeMB, and 2) the total cached bytes is <=
* maxCachedMB */
public NRTCachingDirectory(Directory delegate, double maxMergeSizeMB, double maxCachedMB) {
this.delegate = delegate;
maxMergeSizeBytes = (long) (maxMergeSizeMB*1024*1024);
maxCachedBytes = (long) (maxCachedMB*1024*1024);
}
@Override
public synchronized String[] listAll() throws IOException {
final Set<String> files = new HashSet<String>();
for(String f : cache.listAll()) {
files.add(f);
}
for(String f : delegate.listAll()) {
assert !files.contains(f);
files.add(f);
}
return files.toArray(new String[files.size()]);
}
/** Returns how many bytes are being used by the
* RAMDirectory cache */
public long sizeInBytes() {
return cache.sizeInBytes();
}
@Override
public synchronized boolean fileExists(String name) throws IOException {
return cache.fileExists(name) || delegate.fileExists(name);
}
@Override
public synchronized long fileModified(String name) throws IOException {
if (cache.fileExists(name)) {
return cache.fileModified(name);
} else {
return delegate.fileModified(name);
}
}
@Override
public synchronized void deleteFile(String name) throws IOException {
// Delete from both, in case we are currently uncaching:
if (VERBOSE) {
System.out.println("nrtdir.deleteFile name=" + name);
}
cache.deleteFile(name);
delegate.deleteFile(name);
}
@Override
public synchronized long fileLength(String name) throws IOException {
if (cache.fileExists(name)) {
return cache.fileLength(name);
} else {
return delegate.fileLength(name);
}
}
public String[] listCachedFiles() {
return cache.listAll();
}
@Override
public IndexOutput createOutput(String name) throws IOException {
if (VERBOSE) {
System.out.println("nrtdir.createOutput name=" + name);
}
if (doCacheWrite(name)) {
if (VERBOSE) {
System.out.println(" to cache");
}
return cache.createOutput(name);
} else {
return delegate.createOutput(name);
}
}
@Override
public void sync(Collection<String> fileNames) throws IOException {
if (VERBOSE) {
System.out.println("nrtdir.sync files=" + fileNames);
}
for(String fileName : fileNames) {
unCache(fileName);
}
delegate.sync(fileNames);
}
@Override
public synchronized IndexInput openInput(String name) throws IOException {
if (VERBOSE) {
System.out.println("nrtdir.openInput name=" + name);
}
if (cache.fileExists(name)) {
if (VERBOSE) {
System.out.println(" from cache");
}
return cache.openInput(name);
} else {
return delegate.openInput(name);
}
}
@Override
public synchronized IndexInput openInput(String name, int bufferSize) throws IOException {
if (cache.fileExists(name)) {
return cache.openInput(name, bufferSize);
} else {
return delegate.openInput(name, bufferSize);
}
}
@Override
public Lock makeLock(String name) {
return delegate.makeLock(name);
}
@Override
public void clearLock(String name) throws IOException {
delegate.clearLock(name);
}
/** Close thius directory, which flushes any cached files
* to the delegate and then closes the delegate. */
@Override
public void close() throws IOException {
for(String fileName : cache.listAll()) {
unCache(fileName);
}
cache.close();
delegate.close();
}
private final ConcurrentHashMap<Thread,MergePolicy.OneMerge> merges = new ConcurrentHashMap<Thread,MergePolicy.OneMerge>();
public MergeScheduler getMergeScheduler() {
return new ConcurrentMergeScheduler() {
@Override
protected void doMerge(MergePolicy.OneMerge merge) throws IOException {
try {
merges.put(Thread.currentThread(), merge);
super.doMerge(merge);
} finally {
merges.remove(Thread.currentThread());
}
}
};
}
/** Subclass can override this to customize logic; return
* true if this file should be written to the RAMDirectory. */
protected boolean doCacheWrite(String name) {
final MergePolicy.OneMerge merge = merges.get(Thread.currentThread());
//System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes));
return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes <= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes;
}
private void unCache(String fileName) throws IOException {
final IndexOutput out;
synchronized(this) {
if (!delegate.fileExists(fileName)) {
assert cache.fileExists(fileName);
out = delegate.createOutput(fileName);
} else {
out = null;
}
}
if (out != null) {
IndexInput in = null;
try {
in = cache.openInput(fileName);
in.copyBytes(out, in.length());
} finally {
IOUtils.closeSafely(in, out);
}
synchronized(this) {
cache.deleteFile(fileName);
}
}
}
}

2
lucene/contrib/misc/src/test/org/apache/lucene/index/TestFieldNormModifier.java
View File

@ -61,7 +61,7 @@ public class TestFieldNormModifier extends LuceneTestCase {
super.setUp();
store = newDirectory();
IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < NUM_DOCS; i++) {
Document d = new Document();

2
lucene/contrib/misc/src/test/org/apache/lucene/index/TestIndexSplitter.java
View File

@ -39,7 +39,7 @@ public class TestIndexSplitter extends LuceneTestCase {
mergePolicy.setNoCFSRatio(1);
IndexWriter iw = new IndexWriter(
fsDir,
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).
new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).
setOpenMode(OpenMode.CREATE).
setMergePolicy(mergePolicy)
);

2
lucene/contrib/misc/src/test/org/apache/lucene/index/TestMultiPassIndexSplitter.java
View File

@ -32,7 +32,7 @@ public class TestMultiPassIndexSplitter extends LuceneTestCase {
public void setUp() throws Exception {
super.setUp();
dir = newDirectory();
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
Document doc;
for (int i = 0; i < NUM_DOCS; i++) {
doc = new Document();

2
lucene/contrib/misc/src/test/org/apache/lucene/index/TestTermVectorAccessor.java
View File

@ -25,7 +25,7 @@ public class TestTermVectorAccessor extends LuceneTestCase {
public void test() throws Exception {
Directory dir = newDirectory();
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
IndexWriter iw = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random)));
Document doc;

6
lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
View File

@ -30,7 +30,7 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
@ -134,10 +134,10 @@ public class TestAppendingCodec extends LuceneTestCase {
public void testCodec() throws Exception {
Directory dir = new AppendingRAMDirectory(random, new RAMDirectory());
IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer());
IndexWriterConfig cfg = new IndexWriterConfig(Version.LUCENE_40, new MockAnalyzer(random));
cfg.setCodecProvider(new AppendingCodecProvider());
((LogMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false);
((TieredMergePolicy)cfg.getMergePolicy()).setUseCompoundFile(false);
IndexWriter writer = new IndexWriter(dir, cfg);
Document doc = new Document();
doc.add(newField("f", text, Store.YES, Index.ANALYZED, TermVector.WITH_POSITIONS_OFFSETS));

2
lucene/contrib/misc/src/test/org/apache/lucene/misc/TestHighFreqTerms.java
View File

@ -40,7 +40,7 @@ public class TestHighFreqTerms extends LuceneTestCase {
public static void setUpClass() throws Exception {
dir = newDirectory();
writer = new IndexWriter(dir, newIndexWriterConfig(random,
TEST_VERSION_CURRENT, new MockAnalyzer(MockTokenizer.WHITESPACE, false))
TEST_VERSION_CURRENT, new MockAnalyzer(random, MockTokenizer.WHITESPACE, false))
.setMaxBufferedDocs(2));
writer.setInfoStream(VERBOSE ? System.out : null);
indexDocs(writer);

2
lucene/contrib/misc/src/test/org/apache/lucene/misc/TestLengthNormModifier.java
View File

@ -66,7 +66,7 @@ public class TestLengthNormModifier extends LuceneTestCase {
super.setUp();
store = newDirectory();
IndexWriter writer = new IndexWriter(store, newIndexWriterConfig(
TEST_VERSION_CURRENT, new MockAnalyzer()).setMergePolicy(newInOrderLogMergePolicy()));
TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergePolicy(newLogMergePolicy()));
for (int i = 0; i < NUM_DOCS; i++) {
Document d = new Document();

114
lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java Normal file
View File

@ -0,0 +1,114 @@
package org.apache.lucene.store;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LineFileDocs;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Version;
import org.apache.lucene.util._TestUtil;
public class TestNRTCachingDirectory extends LuceneTestCase {
public void testNRTAndCommit() throws Exception {
Directory dir = newDirectory();
NRTCachingDirectory cachedDir = new NRTCachingDirectory(dir, 2.0, 25.0);
IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
conf.setMergeScheduler(cachedDir.getMergeScheduler());
RandomIndexWriter w = new RandomIndexWriter(random, cachedDir, conf);
w.w.setInfoStream(VERBOSE ? System.out : null);
final LineFileDocs docs = new LineFileDocs(random);
final int numDocs = _TestUtil.nextInt(random, 100, 400);
if (VERBOSE) {
System.out.println("TEST: numDocs=" + numDocs);
}
final List<BytesRef> ids = new ArrayList<BytesRef>();
IndexReader r = null;
for(int docCount=0;docCount<numDocs;docCount++) {
final Document doc = docs.nextDoc();
ids.add(new BytesRef(doc.get("docid")));
w.addDocument(doc);
if (random.nextInt(20) == 17) {
if (r == null) {
r = IndexReader.open(w.w, false);
} else {
final IndexReader r2 = r.reopen();
if (r2 != r) {
r.close();
r = r2;
}
}
assertEquals(1+docCount, r.numDocs());
final IndexSearcher s = new IndexSearcher(r);
// Just make sure search can run; we can't assert
// totHits since it could be 0
TopDocs hits = s.search(new TermQuery(new Term("body", "the")), 10);
// System.out.println("tot hits " + hits.totalHits);
}
}
if (r != null) {
r.close();
}
// Close should force cache to clear since all files are sync'd
w.close();
final String[] cachedFiles = cachedDir.listCachedFiles();
for(String file : cachedFiles) {
System.out.println("FAIL: cached file " + file + " remains after sync");
}
assertEquals(0, cachedFiles.length);
r = IndexReader.open(dir);
for(BytesRef id : ids) {
assertEquals(1, r.docFreq("docid", id));
}
r.close();
cachedDir.close();
}
// NOTE: not a test; just here to make sure the code frag
// in the javadocs is correct!
public void verifyCompiles() throws Exception {
Analyzer analyzer = null;
Directory fsDir = FSDirectory.open(new File("/path/to/index"));
NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 2.0, 25.0);
IndexWriterConfig conf = new IndexWriterConfig(Version.LUCENE_32, analyzer);
conf.setMergeScheduler(cachedFSDir.getMergeScheduler());
IndexWriter writer = new IndexWriter(cachedFSDir, conf);
}
}

25
lucene/contrib/queries/src/java/org/apache/lucene/search/FuzzyLikeThisQuery.java
View File

@ -186,12 +186,13 @@ public class FuzzyLikeThisQuery extends Query
private void addTerms(IndexReader reader,FieldVals f) throws IOException
{
if(f.queryString==null) return;
TokenStream ts=analyzer.tokenStream(f.fieldName,new StringReader(f.queryString));
TokenStream ts=analyzer.reusableTokenStream(f.fieldName,new StringReader(f.queryString));
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
int corpusNumDocs=reader.numDocs();
Term internSavingTemplateTerm =new Term(f.fieldName); //optimization to avoid constructing new Term() objects
HashSet<String> processedTerms=new HashSet<String>();
ts.reset();
while (ts.incrementToken())
{
String term = termAtt.toString();
@ -213,17 +214,15 @@ public class FuzzyLikeThisQuery extends Query
BoostAttribute boostAtt =
fe.attributes().addAttribute(BoostAttribute.class);
while ((possibleMatch = fe.next()) != null) {
if (possibleMatch!=null) {
numVariants++;
totalVariantDocFreqs+=fe.docFreq();
float score=boostAtt.getBoost();
if (variantsQ.size() < MAX_VARIANTS_PER_TERM || score > minScore){
ScoreTerm st=new ScoreTerm(new Term(startTerm.field(), new BytesRef(possibleMatch)),score,startTerm);
variantsQ.insertWithOverflow(st);
minScore = variantsQ.top().score; // maintain minScore
}
maxBoostAtt.setMaxNonCompetitiveBoost(variantsQ.size() >= MAX_VARIANTS_PER_TERM ? minScore : Float.NEGATIVE_INFINITY);
numVariants++;
totalVariantDocFreqs+=fe.docFreq();
float score=boostAtt.getBoost();
if (variantsQ.size() < MAX_VARIANTS_PER_TERM || score > minScore){
ScoreTerm st=new ScoreTerm(new Term(startTerm.field(), new BytesRef(possibleMatch)),score,startTerm);
variantsQ.insertWithOverflow(st);
minScore = variantsQ.top().score; // maintain minScore
}
maxBoostAtt.setMaxNonCompetitiveBoost(variantsQ.size() >= MAX_VARIANTS_PER_TERM ? minScore : Float.NEGATIVE_INFINITY);
}
if(numVariants>0)
@ -246,7 +245,9 @@ public class FuzzyLikeThisQuery extends Query
}
}
}
}
}
ts.end();
ts.close();
}
@Override

6
lucene/contrib/queries/src/java/org/apache/lucene/search/similar/MoreLikeThis.java
View File

@ -881,11 +881,11 @@ public final class MoreLikeThis {
throw new UnsupportedOperationException("To use MoreLikeThis without " +
"term vectors, you must provide an Analyzer");
}
TokenStream ts = analyzer.tokenStream(fieldName, r);
TokenStream ts = analyzer.reusableTokenStream(fieldName, r);
int tokenCount=0;
// for every token
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
String word = termAtt.toString();
tokenCount++;
@ -906,6 +906,8 @@ public final class MoreLikeThis {
cnt.x++;
}
}
ts.end();
ts.close();
}

Some files were not shown because too many files have changed in this diff Show More