SOLR-1256: Show the output of CharFilters in analysis.jsp

git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@792370 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -247,6 +247,9 @@ New Features
     This is an advanced debug log file that can be used to aid developers in fixing IndexWriter bugs. See the commented
     out example in the example solrconfig.xml under the indexDefaults section. 
 	(Chris Harris, Mark Miller)
+
+64. SOLR-1256: Show the output of CharFilters in analysis.jsp. (koji)
+
 Optimizations
 ----------------------
  1. SOLR-374: Use IndexReader.reopen to save resources by re-using parts of the

src/java/org/apache/solr/analysis/CharFilter.java

@@ -60,4 +60,16 @@ public abstract class CharFilter extends CharStream {
   public int read(char[] cbuf, int off, int len) throws IOException {
     return input.read(cbuf, off, len);
   }
+
+  public boolean markSupported(){
+    return input.markSupported();
+  }
+
+  public void mark( int readAheadLimit ) throws IOException {
+    input.mark(readAheadLimit);
+  }
+
+  public void reset() throws IOException {
+    input.reset();
+  }
 }

src/java/org/apache/solr/analysis/CharReader.java

@@ -54,4 +54,16 @@ public final class CharReader extends CharStream {
   public int read(char[] cbuf, int off, int len) throws IOException {
     return input.read(cbuf, off, len );
   }
+
+  public boolean markSupported(){
+    return input.markSupported();
+  }
+
+  public void mark( int readAheadLimit ) throws IOException {
+    input.mark(readAheadLimit);
+  }
+
+  public void reset() throws IOException {
+    input.reset();
+  }
 }

src/java/org/apache/solr/analysis/MappingCharFilter.java

@@ -120,16 +120,4 @@ public class MappingCharFilter extends BaseCharFilter {
     }
     return l == 0 ? -1 : l;
   }
-
-  public boolean markSupported(){
-    return false;
-  }
-
-  public void mark( int readAheadLimit ) throws IOException {
-    throw new IOException( "mark/reset not supported" );
-  }
-
-  public void reset() throws IOException {
-    throw new IOException( "mark/reset not supported" );
-  }
 }

src/test/org/apache/solr/analysis/TestMappingCharFilter.java

@@ -42,6 +42,22 @@ public class TestMappingCharFilter extends BaseTokenTestCase {
     normMap.add( "empty", "" );
   }
 
+  public void testReaderReset() throws Exception {
+    CharStream cs = new MappingCharFilter( normMap, CharReader.get( new StringReader( "x" ) ) );
+    char[] buf = new char[10];
+    int len = cs.read(buf, 0, 10);
+    assertEquals( 1, len );
+    assertEquals( 'x', buf[0]) ;
+    len = cs.read(buf, 0, 10);
+    assertEquals( -1, len );
+
+    // rewind
+    cs.reset();
+    len = cs.read(buf, 0, 10);
+    assertEquals( 1, len );
+    assertEquals( 'x', buf[0]) ;
+  }
+
   public void testNothingChange() throws Exception {
     CharStream cs = new MappingCharFilter( normMap, CharReader.get( new StringReader( "x" ) ) );
     TokenStream ts = new CharStreamAwareWhitespaceTokenizer( cs );

src/webapp/web/admin/analysis.jsp

@@ -19,6 +19,9 @@
                  org.apache.lucene.analysis.Token,
                  org.apache.lucene.analysis.TokenStream,
                  org.apache.lucene.index.Payload,
+                 org.apache.solr.analysis.CharReader,
+                 org.apache.solr.analysis.CharStream,
+                 org.apache.solr.analysis.CharFilterFactory,
                  org.apache.solr.analysis.TokenFilterFactory,
                  org.apache.solr.analysis.TokenizerChain,
                  org.apache.solr.analysis.TokenizerFactory,
@@ -171,19 +174,32 @@
 
 <%!
   private static void doAnalyzer(JspWriter out, SchemaField field, String val, boolean queryAnalyser, boolean verbose, Set<Tok> match) throws Exception {
-    Reader reader = new StringReader(val);
+    CharStream reader = CharReader.get(new StringReader(val));
 
     FieldType ft = field.getType();
      Analyzer analyzer = queryAnalyser ?
              ft.getQueryAnalyzer() : ft.getAnalyzer();
      if (analyzer instanceof TokenizerChain) {
        TokenizerChain tchain = (TokenizerChain)analyzer;
+       CharFilterFactory[] cfiltfacs = tchain.getCharFilterFactories();
        TokenizerFactory tfac = tchain.getTokenizerFactory();
        TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories();
 
-       TokenStream tstream = tfac.create(tchain.charStream(reader));
+       if( cfiltfacs != null ){
+         for(CharFilterFactory cfiltfac : cfiltfacs ){
+           reader = cfiltfac.create(reader);
+           if(verbose){
+             writeHeader(out, cfiltfac.getClass(), cfiltfac.getArgs());
+             writeCharStream(out, reader);
+           }
+         }
+       }
+
+       // StringReader should support reset()
+       reader.reset();
+       TokenStream tstream = tfac.create(reader);
        List<Token> tokens = getTokens(tstream);
-       tstream = tfac.create(tchain.charStream(reader));
+       tstream = tfac.create(reader);
        if (verbose) {
          writeHeader(out, tfac.getClass(), tfac.getArgs());
        }
@@ -453,4 +469,30 @@
     out.println("</table>");
   }
 
+  static void writeCharStream(JspWriter out, CharStream input) throws IOException {
+    out.println("<table width=\"auto\" class=\"analysis\" border=\"1\">");
+    out.println("<tr>");
+
+    out.print("<th NOWRAP>");
+    XML.escapeCharData("text",out);
+    out.println("</th>");
+
+    // StringReader should support reset()
+    input.reset();
+    final int BUFFER_SIZE = 1024;
+    char[] buf = new char[BUFFER_SIZE];
+    int len = 0;
+    StringBuilder sb = new StringBuilder();
+    do {
+      len = input.read( buf, 0, BUFFER_SIZE );
+      sb.append(buf, 0, len);
+    } while( len == BUFFER_SIZE );
+    out.print("<td class=\"debugdata\">");
+    XML.escapeCharData(sb.toString(),out);
+    out.println("</td>");
+    
+    out.println("</tr>");
+    out.println("</table>");
+  }
+
 %>