Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3721: CharFilters were not being invoked in Solr 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1235810 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Yonik Seeley 2012-01-25 16:21:33 +00:00
parent 3f7fd25cc1
commit 1e79c3b98e
2 changed files with 31 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
solr/core/src/java/org/apache/solr/analysis/TokenizerChain.java
View File

@ -19,6 +19,7 @@ package org.apache.solr.analysis;
import org.apache.lucene.analysis.*;
import java.io.IOException;
import java.io.Reader;
/**
@ -48,6 +49,21 @@ public final class TokenizerChain extends SolrAnalyzer {
public TokenizerFactory getTokenizerFactory() { return tokenizer; }
public TokenFilterFactory[] getTokenFilterFactories() { return filters; }
class SolrTokenStreamComponents extends TokenStreamComponents {
public SolrTokenStreamComponents(final Tokenizer source, final TokenStream result) {
super(source, result);
}
@Override
protected void reset(Reader reader) throws IOException {
// the tokenizers are currently reset by the indexing process, so only
// the tokenizer needs to be reset.
Reader r = initReader(reader);
super.reset(r);
}
}
@Override
public Reader initReader(Reader reader) {
if (charFilters != null && charFilters.length > 0) {
@ -62,12 +78,12 @@ public final class TokenizerChain extends SolrAnalyzer {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader aReader) {
Tokenizer tk = tokenizer.create(aReader);
Tokenizer tk = tokenizer.create( initReader(aReader) );
TokenStream ts = tk;
for (TokenFilterFactory filter : filters) {
ts = filter.create(ts);
}
return new TokenStreamComponents(tk, ts);
return new SolrTokenStreamComponents(tk, ts);
}
@Override

13
solr/core/src/test/org/apache/solr/BasicFunctionalityTest.java
View File

@ -222,6 +222,19 @@ public class BasicFunctionalityTest extends SolrTestCaseJ4 {
);
}
@Test
public void testHTMLStrip() {
assertU(add(doc("id","200", "HTMLwhitetok","ABC")));
assertU(add(doc("id","201", "HTMLwhitetok","ABC"))); // do it again to make sure reuse is working
assertU(commit());
assertQ(req("q","HTMLwhitetok:ABC")
,"//*[@numFound='2']"
);
assertQ(req("q","HTMLwhitetok:ABC")
,"//*[@numFound='2']"
);
}
@Test
public void testClientErrorOnMalformedNumbers() throws Exception {