From 5f68c89e2d6138f07c359766b2ada237b28695df Mon Sep 17 00:00:00 2001 From: Uwe Schindler Date: Mon, 3 May 2010 21:37:46 +0000 Subject: [PATCH] LUCENE-2413: Move PerFieldAnalyzerWrapper and LengthFilter git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@940632 13f79535-47bb-0310-9956-ffa450edef68 --- lucene/CHANGES.txt | 2 + .../lucene/analysis/TestKeywordAnalyzer.java | 2 + .../lucene/analysis/TestLengthFilter.java | 41 ---------------- .../analysis/TestPerFieldAnalzyerWrapper.java | 48 ------------------- .../analysis/miscellaneous}/LengthFilter.java | 4 +- .../PerFieldAnalyzerWrapper.java | 4 +- .../miscellaneous}/TestLengthFilter.java | 3 +- .../TestPerFieldAnalzyerWrapper.java | 3 +- .../lucene/analysis/TestKeywordAnalyzer.java | 2 + .../lucene/collation/CollationTestBase.java | 22 +++------ .../solr/analysis/LengthFilterFactory.java | 2 +- 11 files changed, 23 insertions(+), 110 deletions(-) delete mode 100644 lucene/backwards/src/test/org/apache/lucene/analysis/TestLengthFilter.java delete mode 100644 lucene/backwards/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java rename lucene/{src/java/org/apache/lucene/analysis => contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous}/LengthFilter.java (92%) rename lucene/{src/java/org/apache/lucene/analysis => contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous}/PerFieldAnalyzerWrapper.java (96%) rename lucene/{src/test/org/apache/lucene/analysis => contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous}/TestLengthFilter.java (94%) rename lucene/{src/test/org/apache/lucene/analysis => contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous}/TestPerFieldAnalzyerWrapper.java (95%) diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt index 1779565ed1d..a2d03818544 100644 --- a/lucene/CHANGES.txt +++ b/lucene/CHANGES.txt @@ -8,6 +8,8 @@ Changes in backwards compatibility policy - o.a.l.analysis.PorterStemFilter -> o.a.l.analysis.en.PorterStemFilter - o.a.l.analysis.ASCIIFoldingFilter -> o.a.l.analysis.miscellaneous.ASCIIFoldingFilter - o.a.l.analysis.ISOLatin1AccentFilter -> o.a.l.analysis.miscellaneous.ISOLatin1AccentFilter + - o.a.l.analysis.LengthFilter -> o.a.l.analysis.miscellaneous.LengthFilter + - o.a.l.analysis.PerFieldAnalyzerWrapper -> o.a.l.analysis.miscellaneous.PerFieldAnalyzerWrapper ... (in progress) * LUCENE-1458, LUCENE-2111, LUCENE-2354: Changes from flexible indexing: diff --git a/lucene/backwards/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java b/lucene/backwards/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java index 87c1bbe0b0d..48207467e75 100644 --- a/lucene/backwards/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java +++ b/lucene/backwards/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java @@ -56,6 +56,7 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { searcher = new IndexSearcher(directory, true); } + /* public void testPerFieldAnalyzer() throws Exception { PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new SimpleAnalyzer()); analyzer.addAnalyzer("partnum", new KeywordAnalyzer()); @@ -68,6 +69,7 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { "+partnum:Q36 +space", query.toString("description")); assertEquals("doc found!", 1, hits.length); } + */ public void testMutipleDocument() throws Exception { RAMDirectory dir = new RAMDirectory(); diff --git a/lucene/backwards/src/test/org/apache/lucene/analysis/TestLengthFilter.java b/lucene/backwards/src/test/org/apache/lucene/analysis/TestLengthFilter.java deleted file mode 100644 index f6c8bba3804..00000000000 --- a/lucene/backwards/src/test/org/apache/lucene/analysis/TestLengthFilter.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.apache.lucene.analysis; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import org.apache.lucene.analysis.tokenattributes.TermAttribute; - -import java.io.StringReader; - -public class TestLengthFilter extends BaseTokenStreamTestCase { - - public void testFilter() throws Exception { - TokenStream stream = new WhitespaceTokenizer( - new StringReader("short toolong evenmuchlongertext a ab toolong foo")); - LengthFilter filter = new LengthFilter(stream, 2, 6); - TermAttribute termAtt = filter.getAttribute(TermAttribute.class); - - assertTrue(filter.incrementToken()); - assertEquals("short", termAtt.term()); - assertTrue(filter.incrementToken()); - assertEquals("ab", termAtt.term()); - assertTrue(filter.incrementToken()); - assertEquals("foo", termAtt.term()); - assertFalse(filter.incrementToken()); - } - -} diff --git a/lucene/backwards/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java b/lucene/backwards/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java deleted file mode 100644 index 307229069ff..00000000000 --- a/lucene/backwards/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java +++ /dev/null @@ -1,48 +0,0 @@ -package org.apache.lucene.analysis; - -import java.io.StringReader; - -import org.apache.lucene.analysis.tokenattributes.TermAttribute; - -/** - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -public class TestPerFieldAnalzyerWrapper extends BaseTokenStreamTestCase { - public void testPerField() throws Exception { - String text = "Qwerty"; - PerFieldAnalyzerWrapper analyzer = - new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer()); - analyzer.addAnalyzer("special", new SimpleAnalyzer()); - - TokenStream tokenStream = analyzer.tokenStream("field", - new StringReader(text)); - TermAttribute termAtt = tokenStream.getAttribute(TermAttribute.class); - - assertTrue(tokenStream.incrementToken()); - assertEquals("WhitespaceAnalyzer does not lowercase", - "Qwerty", - termAtt.term()); - - tokenStream = analyzer.tokenStream("special", - new StringReader(text)); - termAtt = tokenStream.getAttribute(TermAttribute.class); - assertTrue(tokenStream.incrementToken()); - assertEquals("SimpleAnalyzer lowercases", - "qwerty", - termAtt.term()); - } -} diff --git a/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java b/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java similarity index 92% rename from lucene/src/java/org/apache/lucene/analysis/LengthFilter.java rename to lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java index 551d2d0ae46..75610675bff 100644 --- a/lucene/src/java/org/apache/lucene/analysis/LengthFilter.java +++ b/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/LengthFilter.java @@ -1,4 +1,4 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -19,6 +19,8 @@ package org.apache.lucene.analysis; import java.io.IOException; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.analysis.TokenFilter; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; /** diff --git a/lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java b/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java similarity index 96% rename from lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java rename to lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java index a640c33e30a..30baa90f5c3 100644 --- a/lucene/src/java/org/apache/lucene/analysis/PerFieldAnalyzerWrapper.java +++ b/lucene/contrib/analyzers/common/src/java/org/apache/lucene/analysis/miscellaneous/PerFieldAnalyzerWrapper.java @@ -1,4 +1,4 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,6 +17,8 @@ package org.apache.lucene.analysis; * limitations under the License. */ +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.document.Fieldable; import java.io.Reader; diff --git a/lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java b/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java similarity index 94% rename from lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java rename to lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java index d671bff7dd8..010110c6178 100644 --- a/lucene/src/test/org/apache/lucene/analysis/TestLengthFilter.java +++ b/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLengthFilter.java @@ -1,4 +1,4 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; /** * Licensed to the Apache Software Foundation (ASF) under one or more @@ -17,6 +17,7 @@ package org.apache.lucene.analysis; * limitations under the License. */ +import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; import java.io.StringReader; diff --git a/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java b/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java similarity index 95% rename from lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java rename to lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java index 790e6e8eb33..a06bac83a9f 100644 --- a/lucene/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java +++ b/lucene/contrib/analyzers/common/src/test/org/apache/lucene/analysis/miscellaneous/TestPerFieldAnalzyerWrapper.java @@ -1,7 +1,8 @@ -package org.apache.lucene.analysis; +package org.apache.lucene.analysis.miscellaneous; import java.io.StringReader; +import org.apache.lucene.analysis.*; import org.apache.lucene.analysis.tokenattributes.CharTermAttribute; /** diff --git a/lucene/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java b/lucene/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java index 9550e14275b..712e917ade9 100644 --- a/lucene/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java +++ b/lucene/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java @@ -56,6 +56,7 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { searcher = new IndexSearcher(directory, true); } + /* public void testPerFieldAnalyzer() throws Exception { PerFieldAnalyzerWrapper analyzer = new PerFieldAnalyzerWrapper(new SimpleAnalyzer(TEST_VERSION_CURRENT)); analyzer.addAnalyzer("partnum", new KeywordAnalyzer()); @@ -68,6 +69,7 @@ public class TestKeywordAnalyzer extends BaseTokenStreamTestCase { "+partnum:Q36 +space", query.toString("description")); assertEquals("doc found!", 1, hits.length); } + */ public void testMutipleDocument() throws Exception { RAMDirectory dir = new RAMDirectory(); diff --git a/lucene/src/test/org/apache/lucene/collation/CollationTestBase.java b/lucene/src/test/org/apache/lucene/collation/CollationTestBase.java index 64c70ec1e23..257fe61d300 100644 --- a/lucene/src/test/org/apache/lucene/collation/CollationTestBase.java +++ b/lucene/src/test/org/apache/lucene/collation/CollationTestBase.java @@ -19,7 +19,6 @@ package org.apache.lucene.collation; import org.apache.lucene.analysis.Analyzer; -import org.apache.lucene.analysis.PerFieldAnalyzerWrapper; import org.apache.lucene.analysis.WhitespaceAnalyzer; import org.apache.lucene.store.RAMDirectory; import org.apache.lucene.index.IndexWriter; @@ -40,6 +39,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.util.IndexableBinaryStringTools; import org.apache.lucene.util.LuceneTestCase; +import java.io.StringReader; import java.io.IOException; public abstract class CollationTestBase extends LuceneTestCase { @@ -172,14 +172,8 @@ public abstract class CollationTestBase extends LuceneTestCase { Analyzer denmarkAnalyzer, String usResult) throws Exception { RAMDirectory indexStore = new RAMDirectory(); - PerFieldAnalyzerWrapper analyzer - = new PerFieldAnalyzerWrapper(new WhitespaceAnalyzer(TEST_VERSION_CURRENT)); - analyzer.addAnalyzer("US", usAnalyzer); - analyzer.addAnalyzer("France", franceAnalyzer); - analyzer.addAnalyzer("Sweden", swedenAnalyzer); - analyzer.addAnalyzer("Denmark", denmarkAnalyzer); IndexWriter writer = new IndexWriter(indexStore, new IndexWriterConfig( - TEST_VERSION_CURRENT, analyzer)); + TEST_VERSION_CURRENT, new WhitespaceAnalyzer(TEST_VERSION_CURRENT))); // document data: // the tracer field is used to determine which document was hit @@ -204,17 +198,13 @@ public abstract class CollationTestBase extends LuceneTestCase { doc.add(new Field("contents", sortData[i][1], Field.Store.NO, Field.Index.ANALYZED)); if (sortData[i][2] != null) - doc.add(new Field("US", sortData[i][2], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("US", usAnalyzer.reusableTokenStream("US", new StringReader(sortData[i][2])))); if (sortData[i][3] != null) - doc.add(new Field("France", sortData[i][3], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("France", franceAnalyzer.reusableTokenStream("France", new StringReader(sortData[i][3])))); if (sortData[i][4] != null) - doc.add(new Field("Sweden", sortData[i][4], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("Sweden", swedenAnalyzer.reusableTokenStream("Sweden", new StringReader(sortData[i][4])))); if (sortData[i][5] != null) - doc.add(new Field("Denmark", sortData[i][5], - Field.Store.NO, Field.Index.ANALYZED)); + doc.add(new Field("Denmark", denmarkAnalyzer.reusableTokenStream("Denmark", new StringReader(sortData[i][5])))); writer.addDocument(doc); } writer.optimize(); diff --git a/solr/src/java/org/apache/solr/analysis/LengthFilterFactory.java b/solr/src/java/org/apache/solr/analysis/LengthFilterFactory.java index 25ef94fb7f0..f8105c77709 100644 --- a/solr/src/java/org/apache/solr/analysis/LengthFilterFactory.java +++ b/solr/src/java/org/apache/solr/analysis/LengthFilterFactory.java @@ -18,7 +18,7 @@ package org.apache.solr.analysis; import org.apache.lucene.analysis.TokenStream; -import org.apache.lucene.analysis.LengthFilter; +import org.apache.lucene.analysis.miscellaneous.LengthFilter; import java.util.Map;