From 86dd3a6cdaa619c750ad46a8b7baafb39336eea1 Mon Sep 17 00:00:00 2001 From: Koji Sekiguchi Date: Fri, 7 Aug 2009 23:05:05 +0000 Subject: [PATCH] SOLR-1343: Added HTMLStripCharFilter git-svn-id: https://svn.apache.org/repos/asf/lucene/solr/trunk@802263 13f79535-47bb-0310-9956-ffa450edef68 --- CHANGES.txt | 4 + .../solr/analysis/HTMLStripCharFilter.java | 1353 +++++++++++++++++ .../analysis/HTMLStripCharFilterFactory.java | 29 + .../apache/solr/analysis/HTMLStripReader.java | 1316 +--------------- .../HTMLStripStandardTokenizerFactory.java | 2 + .../HTMLStripWhitespaceTokenizerFactory.java | 2 + ...Test.java => HTMLStripCharFilterTest.java} | 68 +- src/webapp/web/admin/analysis.jsp | 16 +- 8 files changed, 1442 insertions(+), 1348 deletions(-) create mode 100644 src/java/org/apache/solr/analysis/HTMLStripCharFilter.java create mode 100644 src/java/org/apache/solr/analysis/HTMLStripCharFilterFactory.java rename src/test/org/apache/solr/analysis/{HTMLStripReaderTest.java => HTMLStripCharFilterTest.java} (87%) mode change 100755 => 100644 diff --git a/CHANGES.txt b/CHANGES.txt index e9c437d334b..33f7e16c890 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -258,6 +258,10 @@ New Features RequestHandler or SearchComponent to know when a newSearcher or firstSearcher event happened. QuerySenderListender is the only implementation in Solr that implements this, but outside implementations may wish to. See the AbstractSolrEventListener for a helper method. (gsingers) +67. SOLR-1343: Added HTMLStripCharFilter and marked HTMLStripReader, HTMLStripWhitespaceTokenizerFactory and + HTMLStripStandardTokenizerFactory deprecated. To strip HTML tags, HTMLStripCharFilter can be used + with an arbitrary Tokenizer. (koji) + Optimizations ---------------------- diff --git a/src/java/org/apache/solr/analysis/HTMLStripCharFilter.java b/src/java/org/apache/solr/analysis/HTMLStripCharFilter.java new file mode 100644 index 00000000000..33723ead0d0 --- /dev/null +++ b/src/java/org/apache/solr/analysis/HTMLStripCharFilter.java @@ -0,0 +1,1353 @@ +package org.apache.solr.analysis; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.Reader; +import java.util.HashMap; +import java.util.Set; + +import org.apache.lucene.analysis.BaseCharFilter; +import org.apache.lucene.analysis.CharReader; +import org.apache.lucene.analysis.CharStream; + +/** + * A CharFilter that wraps another Reader and attempts to strip out HTML constructs. + * + * @version $Id$ + */ +public class HTMLStripCharFilter extends BaseCharFilter { + private int readAheadLimit = DEFAULT_READ_AHEAD; + private int safeReadAheadLimit = readAheadLimit - 3; + private int numWhitespace = 0; + private int numRead = 0; + private int lastMark; + private Set escapedTags; + + // pushback buffer + private final StringBuilder pushed = new StringBuilder(); + private static final int EOF=-1; + private static final int MISMATCH=-2; + + private static final int MATCH=-3; + // temporary buffer + private final StringBuilder sb = new StringBuilder(); + public static final int DEFAULT_READ_AHEAD = 8192; + + + public static void main(String[] args) throws IOException { + Reader in = new HTMLStripCharFilter( + CharReader.get(new InputStreamReader(System.in))); + int ch; + while ( (ch=in.read()) != -1 ) System.out.print((char)ch); + } + + public HTMLStripCharFilter(CharStream source) { + super(source.markSupported() ? source : CharReader.get(new BufferedReader(source))); + } + + public HTMLStripCharFilter(CharStream source, Set escapedTags){ + this(source); + this.escapedTags = escapedTags; + } + + public HTMLStripCharFilter(CharStream source, Set escapedTags, int readAheadLimit){ + this(source); + this.escapedTags = escapedTags; + this.readAheadLimit = readAheadLimit; + safeReadAheadLimit = readAheadLimit - 3; + } + + public int getReadAheadLimit() { + return readAheadLimit; + } + + private int next() throws IOException { + int len = pushed.length(); + if (len>0) { + int ch = pushed.charAt(len-1); + pushed.setLength(len-1); + return ch; + } + numRead++; + return input.read(); + } + + private int nextSkipWS() throws IOException { + int ch=next(); + while(isSpace(ch)) ch=next(); + return ch; + } + + private int peek() throws IOException { + int len = pushed.length(); + if (len>0) { + return pushed.charAt(len-1); + } + int ch = input.read(); + push(ch); + return ch; + } + + private void push(int ch) { + pushed.append((char)ch); + } + + + private boolean isSpace(int ch) { + switch (ch) { + case ' ': + case '\n': + case '\r': + case '\t': return true; + default: return false; + } + } + + private boolean isHex(int ch) { + return (ch>='0' && ch<='9') || + (ch>='A' && ch<='Z') || + (ch>='a' && ch<='z'); + } + + private boolean isAlpha(int ch) { + return ch>='a' && ch<='z' || ch>='A' && ch<='Z'; + } + + private boolean isDigit(int ch) { + return ch>='0' && ch<='9'; + } + +/*** From HTML 4.0 +[4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | CombiningChar | Extender +[5] Name ::= (Letter | '_' | ':') (NameChar)* +[6] Names ::= Name (#x20 Name)* +[7] Nmtoken ::= (NameChar)+ +[8] Nmtokens ::= Nmtoken (#x20 Nmtoken)* +***/ + + // should I include all id chars allowable by HTML/XML here? + // including accented chars, ':', etc? + private boolean isIdChar(int ch) { + // return Character.isUnicodeIdentifierPart(ch); + // isUnicodeIdentiferPart doesn't include '-'... shoudl I still + // use it and add in '-',':',etc? + return isAlpha(ch) || isDigit(ch) || ch=='.' || + ch=='-' || ch=='_' || ch==':' + || Character.isLetter(ch); + + } + + private boolean isFirstIdChar(int ch) { + return Character.isUnicodeIdentifierStart(ch); + // return isAlpha(ch) || ch=='_' || Character.isLetter(ch); + } + + + private void saveState() throws IOException { + lastMark = numRead; + input.mark(readAheadLimit); + } + + private void restoreState() throws IOException { + input.reset(); + pushed.setLength(0); + } + + private int readNumericEntity() throws IOException { + // "&#" has already been read at this point + + // is this decimal, hex, or nothing at all. + int ch = next(); + int base=10; + boolean invalid=false; + sb.setLength(0); + + if (isDigit(ch)) { + // decimal character entity + sb.append((char)ch); + for (int i=0; i<10; i++) { + ch = next(); + if (isDigit(ch)) { + sb.append((char)ch); + } else { + break; + } + } + } else if (ch=='x') { + // hex character entity + base=16; + sb.setLength(0); + for (int i=0; i<10; i++) { + ch = next(); + if (isHex(ch)) { + sb.append((char)ch); + } else { + break; + } + } + } else { + return MISMATCH; + } + + + // In older HTML, an entity may not have always been terminated + // with a semicolon. We'll also treat EOF or whitespace as terminating + // the entity. + try { + if (ch==';' || ch==-1) { + numWhitespace = sb.length() + 2;// + 2 accounts for &, #, and ;, then, take away 1 for the fact that we do output a char + return Integer.parseInt(sb.toString(), base); + } + + // if whitespace terminated the entity, we need to return + // that whitespace on the next call to read(). + if (isSpace(ch)) { + push(ch); + numWhitespace = sb.length() + 2;// + 2 accounts for &, #, and ;, then, take away 1 for the fact that we do output a char + return Integer.parseInt(sb.toString(), base); + } + } catch (NumberFormatException e) { + return MISMATCH; + } + + // Not an entity... + return MISMATCH; + } + + private int readEntity() throws IOException { + int ch = next(); + if (ch=='#') return readNumericEntity(); + + //read an entity reference + + // for an entity reference, require the ';' for safety. + // otherwise we may try and convert part of some company + // names to an entity. "Alpha&Beta Corp" for instance. + // + // TODO: perhaps I should special case some of the + // more common ones like & to make the ';' optional... + + sb.setLength(0); + sb.append((char)ch); + + for (int i=0; i< safeReadAheadLimit; i++) { + ch=next(); + if (Character.isLetter(ch)) { + sb.append((char)ch); + } else { + break; + } + } + + if (ch==';') { + String entity=sb.toString(); + Character entityChar = entityTable.get(entity); + if (entityChar!=null) { + numWhitespace = entity.length() + 1 ; + return entityChar.charValue(); + } + } + + return MISMATCH; + } + + /*** valid comments according to HTML specs + + + + + + Hello --> + + #comments inside of an entity decl: + + + Turns out, IE & mozilla don't parse comments correctly. + Since this is meant to be a practical stripper, I'll just + try and duplicate what the browsers do. + + + + + + + ***/ + + private int readBang(boolean inScript) throws IOException { + // at this point, "' ) { + + int ch = next(); + if (ch=='>') return MATCH; + + // if it starts with " + //since we did readComment already, it may be the case that we are already deep into the read ahead buffer + //so, we may need to abort sooner + while ((numRead - lastMark) < safeReadAheadLimit) { + ch = next(); + if (ch=='>') { + return MATCH; + } + else if (ch<0) { + return MISMATCH; + } + } + } + return MISMATCH; + } + + // tries to read comments the way browsers do, not + // strictly by the standards. + // + // GRRRR. it turns out that in the wild, a + // + private int readComment(boolean inScript) throws IOException { + // at this point "') { + push(ch); + push('-'); + continue; + } + + return MATCH; + } else if ((ch=='\'' || ch=='"') && inScript) { + push(ch); + int ret=readScriptString(); + // if this wasn't a string, there's not much we can do + // at this point without having a stack of stream states in + // order to "undo" just the latest. + } else if (ch=='<') { + eatSSI(); + } + + } + return MISMATCH; + + } + + + + private int readTag() throws IOException { + // at this point '<' has already been read + int ch = next(); + if (!isAlpha(ch)) { + push(ch); + return MISMATCH; + } + + sb.setLength(0); + sb.append((char)ch); + while((numRead - lastMark) < safeReadAheadLimit) { + + ch = next(); + if (isIdChar(ch)) { + sb.append((char)ch); + } else if (ch=='/') { + // Hmmm, a tag can close with "/>" as well as "/ >" + // read end tag '/>' or '/ >', etc + return nextSkipWS()=='>' ? MATCH : MISMATCH; + } else { + break; + } + } + if (escapedTags!=null && escapedTags.contains(sb.toString())){ + //if this is a reservedTag, then keep it + return MISMATCH; + } + // After the tag id, there needs to be either whitespace or + // '>' + if ( !(ch=='>' || isSpace(ch)) ) { + return MISMATCH; + } + + if (ch!='>') { + // process attributes + while ((numRead - lastMark) < safeReadAheadLimit) { + ch=next(); + if (isSpace(ch)) { + continue; + } else if (isFirstIdChar(ch)) { + push(ch); + int ret = readAttr2(); + if (ret==MISMATCH) return ret; + } else if (ch=='/') { + // read end tag '/>' or '/ >', etc + return nextSkipWS()=='>' ? MATCH : MISMATCH; + } else if (ch=='>') { + break; + } else { + return MISMATCH; + } + + } + if ((numRead - lastMark) >= safeReadAheadLimit){ + return MISMATCH;//exit out if we exceeded the buffer + } + } + + // We only get to this point after we have read the + // entire tag. Now let's see if it's a special tag. + String name=sb.toString(); + if (name.equalsIgnoreCase("script") || name.equalsIgnoreCase("style")) { + // The content of script and style elements is + // CDATA in HTML 4 but PCDATA in XHTML. + + /* From HTML4: + Although the STYLE and SCRIPT elements use CDATA for their data model, + for these elements, CDATA must be handled differently by user agents. + Markup and entities must be treated as raw text and passed to the application + as is. The first occurrence of the character sequence "foo + // beware markup in script strings: ...document.write("")foo + // TODO: do I need to worry about CDATA sections "') return MISMATCH; + return MATCH; + } else if (ch=='\'' || ch=='"') { + // read javascript string to avoid a false match. + push(ch); + int ret = readScriptString(); + // what to do about a non-match (non-terminated string?) + // play it safe and index the rest of the data I guess... + if (ret==MISMATCH) return MISMATCH; + } else if (ch<0) { + return MISMATCH; + } + + } + return MISMATCH; + } + + + // read a string escaped by backslashes + private int readScriptString() throws IOException { + int quoteChar = next(); + if (quoteChar!='\'' && quoteChar!='"') return MISMATCH; + + while((numRead - lastMark) < safeReadAheadLimit) { + int ch = next(); + if (ch==quoteChar) return MATCH; + else if (ch=='\\') { + ch=next(); + } else if (ch<0) { + return MISMATCH; + } else if (ch=='<') { + eatSSI(); + } + + } + return MISMATCH; + } + + + private int readName(boolean checkEscaped) throws IOException { + StringBuilder builder = (checkEscaped && escapedTags!=null) ? new StringBuilder() : null; + int ch = read(); + if (builder!=null) builder.append((char)ch); + if (!isFirstIdChar(ch)) return MISMATCH; + ch = read(); + if (builder!=null) builder.append((char)ch); + while(isIdChar(ch)) { + ch=read(); + if (builder!=null) builder.append((char)ch); + } + if (ch!=-1) { + push(ch); + + } + //strip off the trailing > + if (builder!=null && escapedTags.contains(builder.substring(0, builder.length() - 1))){ + return MISMATCH; + } + return MATCH; + } + + /*** + [10] AttValue ::= '"' ([^<&"] | Reference)* '"' + | "'" ([^<&'] | Reference)* "'" + + need to also handle unquoted attributes, and attributes w/o values: + + + ***/ + + // This reads attributes and attempts to handle any + // embedded server side includes that would otherwise + // mess up the quote handling. + // "> + private int readAttr2() throws IOException { + if ((numRead - lastMark < safeReadAheadLimit)) { + int ch = read(); + if (!isFirstIdChar(ch)) return MISMATCH; + ch = read(); + while(isIdChar(ch) && ((numRead - lastMark) < safeReadAheadLimit)){ + ch=read(); + } + if (isSpace(ch)) ch = nextSkipWS(); + + // attributes may not have a value at all! + // if (ch != '=') return MISMATCH; + if (ch != '=') { + push(ch); + return MATCH; + } + + int quoteChar = nextSkipWS(); + + if (quoteChar=='"' || quoteChar=='\'') { + while ((numRead - lastMark) < safeReadAheadLimit) { + ch = next(); + if (ch<0) return MISMATCH; + else if (ch=='<') { + eatSSI(); + } + else if (ch==quoteChar) { + return MATCH; + //} else if (ch=='<') { + // return MISMATCH; + } + + } + } else { + // unquoted attribute + while ((numRead - lastMark) < safeReadAheadLimit) { + ch = next(); + if (ch<0) return MISMATCH; + else if (isSpace(ch)) { + push(ch); + return MATCH; + } else if (ch=='>') { + push(ch); + return MATCH; + } else if (ch=='<') { + eatSSI(); + } + + } + } + } + return MISMATCH; + } + + // skip past server side include + private int eatSSI() throws IOException { + // at this point, only a "<" was read. + // on a mismatch, push back the last char so that if it was + // a quote that closes the attribute, it will be re-read and matched. + int ch = next(); + if (ch!='!') { + push(ch); + return MISMATCH; + } + ch=next(); + if (ch!='-') { + push(ch); + return MISMATCH; + } + ch=next(); + if (ch!='-') { + push(ch); + return MISMATCH; + } + ch=next(); + if (ch!='#') { + push(ch); + return MISMATCH; + } + + push('#'); push('-'); push('-'); + return readComment(false); + } + + private int readProcessingInstruction() throws IOException { + // "') { + next(); + return MATCH; + } else if (ch==-1) { + return MISMATCH; + } + + } + return MISMATCH; + } + + + + public int read() throws IOException { + // TODO: Do we ever want to preserve CDATA sections? + // where do we have to worry about them? + // + if (numWhitespace > 0){ + numWhitespace--; + return ' '; + } + //do not limit this one by the READAHEAD + while(true) { + int lastNumRead = numRead; + int ch = next(); + + switch (ch) { + case '&': + saveState(); + ch = readEntity(); + if (ch>=0) return ch; + if (ch==MISMATCH) { + restoreState(); + + return '&'; + } + break; + + case '<': + saveState(); + ch = next(); + int ret = MISMATCH; + if (ch=='!') { + ret = readBang(false); + } else if (ch=='/') { + ret = readName(true); + if (ret==MATCH) { + ch=nextSkipWS(); + ret= ch=='>' ? MATCH : MISMATCH; + } + } else if (isAlpha(ch)) { + push(ch); + ret = readTag(); + } else if (ch=='?') { + ret = readProcessingInstruction(); + } + + // matched something to be discarded, so break + // from this case and continue in the loop + if (ret==MATCH) { + //break;//was + //return whitespace from + numWhitespace = (numRead - lastNumRead) - 1;//tack on the -1 since we are returning a space right now + return ' '; + } + + // didn't match any HTML constructs, so roll back + // the stream state and just return '<' + restoreState(); + return '<'; + + default: return ch; + } + + } + + + } + + public int read(char cbuf[], int off, int len) throws IOException { + int i=0; + for (i=0; i entityTable; + static { + entityTable = new HashMap(); + // entityName and entityVal generated from the python script + // included in comments at the end of this file. + final String[] entityName={ "zwnj","aring","gt","yen","ograve","Chi","delta","rang","sup","trade","Ntilde","xi","upsih","nbsp","Atilde","radic","otimes","aelig","oelig","equiv","ni","infin","Psi","auml","cup","Epsilon","otilde","lt","Icirc","Eacute","Lambda","sbquo","Prime","prime","psi","Kappa","rsaquo","Tau","uacute","ocirc","lrm","zwj","cedil","Alpha","not","amp","AElig","oslash","acute","lceil","alefsym","laquo","shy","loz","ge","Igrave","nu","Ograve","lsaquo","sube","euro","rarr","sdot","rdquo","Yacute","lfloor","lArr","Auml","Dagger","brvbar","Otilde","szlig","clubs","diams","agrave","Ocirc","Iota","Theta","Pi","zeta","Scaron","frac14","egrave","sub","iexcl","frac12","ordf","sum","prop","Uuml","ntilde","atilde","asymp","uml","prod","nsub","reg","rArr","Oslash","emsp","THORN","yuml","aacute","Mu","hArr","le","thinsp","dArr","ecirc","bdquo","Sigma","Aring","tilde","nabla","mdash","uarr","times","Ugrave","Eta","Agrave","chi","real","circ","eth","rceil","iuml","gamma","lambda","harr","Egrave","frac34","dagger","divide","Ouml","image","ndash","hellip","igrave","Yuml","ang","alpha","frasl","ETH","lowast","Nu","plusmn","bull","sup1","sup2","sup3","Aacute","cent","oline","Beta","perp","Delta","there4","pi","iota","empty","euml","notin","iacute","para","epsilon","weierp","OElig","uuml","larr","icirc","Upsilon","omicron","upsilon","copy","Iuml","Oacute","Xi","kappa","ccedil","Ucirc","cap","mu","scaron","lsquo","isin","Zeta","minus","deg","and","tau","pound","curren","int","ucirc","rfloor","ensp","crarr","ugrave","exist","cong","theta","oplus","permil","Acirc","piv","Euml","Phi","Iacute","quot","Uacute","Omicron","ne","iquest","eta","rsquo","yacute","Rho","darr","Ecirc","Omega","acirc","sim","phi","sigmaf","macr","thetasym","Ccedil","ordm","uArr","forall","beta","fnof","rho","micro","eacute","omega","middot","Gamma","rlm","lang","spades","supe","thorn","ouml","or","raquo","part","sect","ldquo","hearts","sigma","oacute"}; + final char[] entityVal={ 8204,229,62,165,242,935,948,9002,8835,8482,209,958,978,160,195,8730,8855,230,339,8801,8715,8734,936,228,8746,917,245,60,206,201,923,8218,8243,8242,968,922,8250,932,250,244,8206,8205,184,913,172,38,198,248,180,8968,8501,171,173,9674,8805,204,957,210,8249,8838,8364,8594,8901,8221,221,8970,8656,196,8225,166,213,223,9827,9830,224,212,921,920,928,950,352,188,232,8834,161,189,170,8721,8733,220,241,227,8776,168,8719,8836,174,8658,216,8195,222,255,225,924,8660,8804,8201,8659,234,8222,931,197,732,8711,8212,8593,215,217,919,192,967,8476,710,240,8969,239,947,955,8596,200,190,8224,247,214,8465,8211,8230,236,376,8736,945,8260,208,8727,925,177,8226,185,178,179,193,162,8254,914,8869,916,8756,960,953,8709,235,8713,237,182,949,8472,338,252,8592,238,933,959,965,169,207,211,926,954,231,219,8745,956,353,8216,8712,918,8722,176,8743,964,163,164,8747,251,8971,8194,8629,249,8707,8773,952,8853,8240,194,982,203,934,205,34,218,927,8800,191,951,8217,253,929,8595,202,937,226,8764,966,962,175,977,199,186,8657,8704,946,402,961,181,233,969,183,915,8207,9001,9824,8839,254,246,8744,187,8706,167,8220,9829,963,243}; + for (int i=0; i + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +24.3 Character entity references for symbols, mathematical symbols, and Greek letters + +The character entity references in this section produce characters that may be represented by glyphs in the widely available Adobe Symbol font, including Greek characters, various bracketing symbols, and a selection of mathematical operators such as gradient, product, and summation symbols. + +To support these entities, user agents may support full [ISO10646] or use other means. Display of glyphs for these characters may be obtained by being able to display the relevant [ISO10646] characters or by other means, such as internally mapping the listed entities, numeric character references, and characters to the appropriate position in some font that contains the requisite glyphs. + +When to use Greek entities. This entity set contains all the letters used in modern Greek. However, it does not include Greek punctuation, precomposed accented characters nor the non-spacing accents (tonos, dialytika) required to compose them. There are no archaic letters, Coptic-unique letters, or precomposed letters for Polytonic Greek. The entities defined here are not intended for the representation of modern Greek text and would not be an efficient representation; rather, they are intended for occasional Greek letters used in technical and mathematical works. +24.3.1 The list of characters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +24.4 Character entity references for markup-significant and internationalization characters + +The character entity references in this section are for escaping markup-significant characters (these are the same as those in HTML 2.0 and 3.2), for denoting spaces and dashes. Other characters in this section apply to internationalization issues such as the disambiguation of bidirectional text (see the section on bidirectional text for details). + +Entities have also been added for the remaining characters occurring in CP-1252 which do not occur in the HTMLlat1 or HTMLsymbol entity sets. These all occur in the 128 to 159 range within the CP-1252 charset. These entities permit the characters to be denoted in a platform-independent manner. + +To support these entities, user agents may support full [ISO10646] or use other means. Display of glyphs for these characters may be obtained by being able to display the relevant [ISO10646] characters or by other means, such as internally mapping the listed entities, numeric character references, and characters to the appropriate position in some font that contains the requisite glyphs. +24.4.1 The list of characters + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + +codes={} +for line in text.split('\n'): + parts = line.split() + if len(parts)<3 or parts[0]!=' escapedTags; - - // pushback buffer - private final StringBuilder pushed = new StringBuilder(); - private static final int EOF=-1; - private static final int MISMATCH=-2; - - private static final int MATCH=-3; - // temporary buffer - private final StringBuilder sb = new StringBuilder(); - public static final int DEFAULT_READ_AHEAD = 8192; - +@Deprecated +public class HTMLStripReader extends HTMLStripCharFilter { public static void main(String[] args) throws IOException { Reader in = new HTMLStripReader( @@ -59,1296 +44,15 @@ public class HTMLStripReader extends Reader { while ( (ch=in.read()) != -1 ) System.out.print((char)ch); } - public HTMLStripReader(Reader source) { - super(); - this.in=source.markSupported() ? source : new BufferedReader(source); + public HTMLStripReader(Reader source){ + super(CharReader.get(source.markSupported() ? source : new BufferedReader(source))); } public HTMLStripReader(Reader source, Set escapedTags){ - this(source); - this.escapedTags = escapedTags; + super(CharReader.get(source),escapedTags); } - public HTMLStripReader(Reader source, Set escapedTags, int readAheadLimit){ - this(source); - this.escapedTags = escapedTags; - this.readAheadLimit = readAheadLimit; - safeReadAheadLimit = readAheadLimit - 3; + public HTMLStripReader(Reader source, Set escapedTags,int readAheadLimit){ + super(CharReader.get(source),escapedTags,readAheadLimit); } - - public int getReadAheadLimit() { - return readAheadLimit; - } - - private int next() throws IOException { - int len = pushed.length(); - if (len>0) { - int ch = pushed.charAt(len-1); - pushed.setLength(len-1); - return ch; - } - numRead++; - return in.read(); - } - - private int nextSkipWS() throws IOException { - int ch=next(); - while(isSpace(ch)) ch=next(); - return ch; - } - - private int peek() throws IOException { - int len = pushed.length(); - if (len>0) { - return pushed.charAt(len-1); - } - int ch = in.read(); - push(ch); - return ch; - } - - private void push(int ch) { - pushed.append((char)ch); - } - - - private boolean isSpace(int ch) { - switch (ch) { - case ' ': - case '\n': - case '\r': - case '\t': return true; - default: return false; - } - } - - private boolean isHex(int ch) { - return (ch>='0' && ch<='9') || - (ch>='A' && ch<='Z') || - (ch>='a' && ch<='z'); - } - - private boolean isAlpha(int ch) { - return ch>='a' && ch<='z' || ch>='A' && ch<='Z'; - } - - private boolean isDigit(int ch) { - return ch>='0' && ch<='9'; - } - -/*** From HTML 4.0 -[4] NameChar ::= Letter | Digit | '.' | '-' | '_' | ':' | CombiningChar | Extender -[5] Name ::= (Letter | '_' | ':') (NameChar)* -[6] Names ::= Name (#x20 Name)* -[7] Nmtoken ::= (NameChar)+ -[8] Nmtokens ::= Nmtoken (#x20 Nmtoken)* -***/ - - // should I include all id chars allowable by HTML/XML here? - // including accented chars, ':', etc? - private boolean isIdChar(int ch) { - // return Character.isUnicodeIdentifierPart(ch); - // isUnicodeIdentiferPart doesn't include '-'... shoudl I still - // use it and add in '-',':',etc? - return isAlpha(ch) || isDigit(ch) || ch=='.' || - ch=='-' || ch=='_' || ch==':' - || Character.isLetter(ch); - - } - - private boolean isFirstIdChar(int ch) { - return Character.isUnicodeIdentifierStart(ch); - // return isAlpha(ch) || ch=='_' || Character.isLetter(ch); - } - - - private void saveState() throws IOException { - lastMark = numRead; - in.mark(readAheadLimit); - } - - private void restoreState() throws IOException { - in.reset(); - pushed.setLength(0); - } - - private int readNumericEntity() throws IOException { - // "&#" has already been read at this point - - // is this decimal, hex, or nothing at all. - int ch = next(); - int base=10; - boolean invalid=false; - sb.setLength(0); - - if (isDigit(ch)) { - // decimal character entity - sb.append((char)ch); - for (int i=0; i<10; i++) { - ch = next(); - if (isDigit(ch)) { - sb.append((char)ch); - } else { - break; - } - } - } else if (ch=='x') { - // hex character entity - base=16; - sb.setLength(0); - for (int i=0; i<10; i++) { - ch = next(); - if (isHex(ch)) { - sb.append((char)ch); - } else { - break; - } - } - } else { - return MISMATCH; - } - - - // In older HTML, an entity may not have always been terminated - // with a semicolon. We'll also treat EOF or whitespace as terminating - // the entity. - try { - if (ch==';' || ch==-1) { - numWhitespace = sb.length() + 2;// + 2 accounts for &, #, and ;, then, take away 1 for the fact that we do output a char - return Integer.parseInt(sb.toString(), base); - } - - // if whitespace terminated the entity, we need to return - // that whitespace on the next call to read(). - if (isSpace(ch)) { - push(ch); - numWhitespace = sb.length() + 2;// + 2 accounts for &, #, and ;, then, take away 1 for the fact that we do output a char - return Integer.parseInt(sb.toString(), base); - } - } catch (NumberFormatException e) { - return MISMATCH; - } - - // Not an entity... - return MISMATCH; - } - - private int readEntity() throws IOException { - int ch = next(); - if (ch=='#') return readNumericEntity(); - - //read an entity reference - - // for an entity reference, require the ';' for safety. - // otherwise we may try and convert part of some company - // names to an entity. "Alpha&Beta Corp" for instance. - // - // TODO: perhaps I should special case some of the - // more common ones like & to make the ';' optional... - - sb.setLength(0); - sb.append((char)ch); - - for (int i=0; i< safeReadAheadLimit; i++) { - ch=next(); - if (Character.isLetter(ch)) { - sb.append((char)ch); - } else { - break; - } - } - - if (ch==';') { - String entity=sb.toString(); - Character entityChar = entityTable.get(entity); - if (entityChar!=null) { - numWhitespace = entity.length() + 1 ; - return entityChar.charValue(); - } - } - - return MISMATCH; - } - - /*** valid comments according to HTML specs - - - - - - Hello --> - - #comments inside of an entity decl: - - - Turns out, IE & mozilla don't parse comments correctly. - Since this is meant to be a practical stripper, I'll just - try and duplicate what the browsers do. - - - - - - - ***/ - - private int readBang(boolean inScript) throws IOException { - // at this point, "' ) { - - int ch = next(); - if (ch=='>') return MATCH; - - // if it starts with " - //since we did readComment already, it may be the case that we are already deep into the read ahead buffer - //so, we may need to abort sooner - while ((numRead - lastMark) < safeReadAheadLimit) { - ch = next(); - if (ch=='>') { - return MATCH; - } - else if (ch<0) { - return MISMATCH; - } - } - } - return MISMATCH; - } - - // tries to read comments the way browsers do, not - // strictly by the standards. - // - // GRRRR. it turns out that in the wild, a - // - private int readComment(boolean inScript) throws IOException { - // at this point "') { - push(ch); - push('-'); - continue; - } - - return MATCH; - } else if ((ch=='\'' || ch=='"') && inScript) { - push(ch); - int ret=readScriptString(); - // if this wasn't a string, there's not much we can do - // at this point without having a stack of stream states in - // order to "undo" just the latest. - } else if (ch=='<') { - eatSSI(); - } - - } - return MISMATCH; - - } - - - - private int readTag() throws IOException { - // at this point '<' has already been read - int ch = next(); - if (!isAlpha(ch)) { - push(ch); - return MISMATCH; - } - - sb.setLength(0); - sb.append((char)ch); - while((numRead - lastMark) < safeReadAheadLimit) { - - ch = next(); - if (isIdChar(ch)) { - sb.append((char)ch); - } else if (ch=='/') { - // Hmmm, a tag can close with "/>" as well as "/ >" - // read end tag '/>' or '/ >', etc - return nextSkipWS()=='>' ? MATCH : MISMATCH; - } else { - break; - } - } - if (escapedTags!=null && escapedTags.contains(sb.toString())){ - //if this is a reservedTag, then keep it - return MISMATCH; - } - // After the tag id, there needs to be either whitespace or - // '>' - if ( !(ch=='>' || isSpace(ch)) ) { - return MISMATCH; - } - - if (ch!='>') { - // process attributes - while ((numRead - lastMark) < safeReadAheadLimit) { - ch=next(); - if (isSpace(ch)) { - continue; - } else if (isFirstIdChar(ch)) { - push(ch); - int ret = readAttr2(); - if (ret==MISMATCH) return ret; - } else if (ch=='/') { - // read end tag '/>' or '/ >', etc - return nextSkipWS()=='>' ? MATCH : MISMATCH; - } else if (ch=='>') { - break; - } else { - return MISMATCH; - } - - } - if ((numRead - lastMark) >= safeReadAheadLimit){ - return MISMATCH;//exit out if we exceeded the buffer - } - } - - // We only get to this point after we have read the - // entire tag. Now let's see if it's a special tag. - String name=sb.toString(); - if (name.equalsIgnoreCase("script") || name.equalsIgnoreCase("style")) { - // The content of script and style elements is - // CDATA in HTML 4 but PCDATA in XHTML. - - /* From HTML4: - Although the STYLE and SCRIPT elements use CDATA for their data model, - for these elements, CDATA must be handled differently by user agents. - Markup and entities must be treated as raw text and passed to the application - as is. The first occurrence of the character sequence "foo - // beware markup in script strings: ...document.write("")foo - // TODO: do I need to worry about CDATA sections "') return MISMATCH; - return MATCH; - } else if (ch=='\'' || ch=='"') { - // read javascript string to avoid a false match. - push(ch); - int ret = readScriptString(); - // what to do about a non-match (non-terminated string?) - // play it safe and index the rest of the data I guess... - if (ret==MISMATCH) return MISMATCH; - } else if (ch<0) { - return MISMATCH; - } - - } - return MISMATCH; - } - - - // read a string escaped by backslashes - private int readScriptString() throws IOException { - int quoteChar = next(); - if (quoteChar!='\'' && quoteChar!='"') return MISMATCH; - - while((numRead - lastMark) < safeReadAheadLimit) { - int ch = next(); - if (ch==quoteChar) return MATCH; - else if (ch=='\\') { - ch=next(); - } else if (ch<0) { - return MISMATCH; - } else if (ch=='<') { - eatSSI(); - } - - } - return MISMATCH; - } - - - private int readName(boolean checkEscaped) throws IOException { - StringBuilder builder = (checkEscaped && escapedTags!=null) ? new StringBuilder() : null; - int ch = read(); - if (builder!=null) builder.append((char)ch); - if (!isFirstIdChar(ch)) return MISMATCH; - ch = read(); - if (builder!=null) builder.append((char)ch); - while(isIdChar(ch)) { - ch=read(); - if (builder!=null) builder.append((char)ch); - } - if (ch!=-1) { - push(ch); - - } - //strip off the trailing > - if (builder!=null && escapedTags.contains(builder.substring(0, builder.length() - 1))){ - return MISMATCH; - } - return MATCH; - } - - /*** - [10] AttValue ::= '"' ([^<&"] | Reference)* '"' - | "'" ([^<&'] | Reference)* "'" - - need to also handle unquoted attributes, and attributes w/o values: - - - ***/ - - // This reads attributes and attempts to handle any - // embedded server side includes that would otherwise - // mess up the quote handling. - // "> - private int readAttr2() throws IOException { - if ((numRead - lastMark < safeReadAheadLimit)) { - int ch = read(); - if (!isFirstIdChar(ch)) return MISMATCH; - ch = read(); - while(isIdChar(ch) && ((numRead - lastMark) < safeReadAheadLimit)){ - ch=read(); - } - if (isSpace(ch)) ch = nextSkipWS(); - - // attributes may not have a value at all! - // if (ch != '=') return MISMATCH; - if (ch != '=') { - push(ch); - return MATCH; - } - - int quoteChar = nextSkipWS(); - - if (quoteChar=='"' || quoteChar=='\'') { - while ((numRead - lastMark) < safeReadAheadLimit) { - ch = next(); - if (ch<0) return MISMATCH; - else if (ch=='<') { - eatSSI(); - } - else if (ch==quoteChar) { - return MATCH; - //} else if (ch=='<') { - // return MISMATCH; - } - - } - } else { - // unquoted attribute - while ((numRead - lastMark) < safeReadAheadLimit) { - ch = next(); - if (ch<0) return MISMATCH; - else if (isSpace(ch)) { - push(ch); - return MATCH; - } else if (ch=='>') { - push(ch); - return MATCH; - } else if (ch=='<') { - eatSSI(); - } - - } - } - } - return MISMATCH; - } - - // skip past server side include - private int eatSSI() throws IOException { - // at this point, only a "<" was read. - // on a mismatch, push back the last char so that if it was - // a quote that closes the attribute, it will be re-read and matched. - int ch = next(); - if (ch!='!') { - push(ch); - return MISMATCH; - } - ch=next(); - if (ch!='-') { - push(ch); - return MISMATCH; - } - ch=next(); - if (ch!='-') { - push(ch); - return MISMATCH; - } - ch=next(); - if (ch!='#') { - push(ch); - return MISMATCH; - } - - push('#'); push('-'); push('-'); - return readComment(false); - } - - private int readProcessingInstruction() throws IOException { - // "') { - next(); - return MATCH; - } else if (ch==-1) { - return MISMATCH; - } - - } - return MISMATCH; - } - - - - public int read() throws IOException { - // TODO: Do we ever want to preserve CDATA sections? - // where do we have to worry about them? - // - if (numWhitespace > 0){ - numWhitespace--; - return ' '; - } - //do not limit this one by the READAHEAD - while(true) { - int lastNumRead = numRead; - int ch = next(); - - switch (ch) { - case '&': - saveState(); - ch = readEntity(); - if (ch>=0) return ch; - if (ch==MISMATCH) { - restoreState(); - - return '&'; - } - break; - - case '<': - saveState(); - ch = next(); - int ret = MISMATCH; - if (ch=='!') { - ret = readBang(false); - } else if (ch=='/') { - ret = readName(true); - if (ret==MATCH) { - ch=nextSkipWS(); - ret= ch=='>' ? MATCH : MISMATCH; - } - } else if (isAlpha(ch)) { - push(ch); - ret = readTag(); - } else if (ch=='?') { - ret = readProcessingInstruction(); - } - - // matched something to be discarded, so break - // from this case and continue in the loop - if (ret==MATCH) { - //break;//was - //return whitespace from - numWhitespace = (numRead - lastNumRead) - 1;//tack on the -1 since we are returning a space right now - return ' '; - } - - // didn't match any HTML constructs, so roll back - // the stream state and just return '<' - restoreState(); - return '<'; - - default: return ch; - } - - } - - - } - - public int read(char cbuf[], int off, int len) throws IOException { - int i=0; - for (i=0; i entityTable; - static { - entityTable = new HashMap(); - // entityName and entityVal generated from the python script - // included in comments at the end of this file. - final String[] entityName={ "zwnj","aring","gt","yen","ograve","Chi","delta","rang","sup","trade","Ntilde","xi","upsih","nbsp","Atilde","radic","otimes","aelig","oelig","equiv","ni","infin","Psi","auml","cup","Epsilon","otilde","lt","Icirc","Eacute","Lambda","sbquo","Prime","prime","psi","Kappa","rsaquo","Tau","uacute","ocirc","lrm","zwj","cedil","Alpha","not","amp","AElig","oslash","acute","lceil","alefsym","laquo","shy","loz","ge","Igrave","nu","Ograve","lsaquo","sube","euro","rarr","sdot","rdquo","Yacute","lfloor","lArr","Auml","Dagger","brvbar","Otilde","szlig","clubs","diams","agrave","Ocirc","Iota","Theta","Pi","zeta","Scaron","frac14","egrave","sub","iexcl","frac12","ordf","sum","prop","Uuml","ntilde","atilde","asymp","uml","prod","nsub","reg","rArr","Oslash","emsp","THORN","yuml","aacute","Mu","hArr","le","thinsp","dArr","ecirc","bdquo","Sigma","Aring","tilde","nabla","mdash","uarr","times","Ugrave","Eta","Agrave","chi","real","circ","eth","rceil","iuml","gamma","lambda","harr","Egrave","frac34","dagger","divide","Ouml","image","ndash","hellip","igrave","Yuml","ang","alpha","frasl","ETH","lowast","Nu","plusmn","bull","sup1","sup2","sup3","Aacute","cent","oline","Beta","perp","Delta","there4","pi","iota","empty","euml","notin","iacute","para","epsilon","weierp","OElig","uuml","larr","icirc","Upsilon","omicron","upsilon","copy","Iuml","Oacute","Xi","kappa","ccedil","Ucirc","cap","mu","scaron","lsquo","isin","Zeta","minus","deg","and","tau","pound","curren","int","ucirc","rfloor","ensp","crarr","ugrave","exist","cong","theta","oplus","permil","Acirc","piv","Euml","Phi","Iacute","quot","Uacute","Omicron","ne","iquest","eta","rsquo","yacute","Rho","darr","Ecirc","Omega","acirc","sim","phi","sigmaf","macr","thetasym","Ccedil","ordm","uArr","forall","beta","fnof","rho","micro","eacute","omega","middot","Gamma","rlm","lang","spades","supe","thorn","ouml","or","raquo","part","sect","ldquo","hearts","sigma","oacute"}; - final char[] entityVal={ 8204,229,62,165,242,935,948,9002,8835,8482,209,958,978,160,195,8730,8855,230,339,8801,8715,8734,936,228,8746,917,245,60,206,201,923,8218,8243,8242,968,922,8250,932,250,244,8206,8205,184,913,172,38,198,248,180,8968,8501,171,173,9674,8805,204,957,210,8249,8838,8364,8594,8901,8221,221,8970,8656,196,8225,166,213,223,9827,9830,224,212,921,920,928,950,352,188,232,8834,161,189,170,8721,8733,220,241,227,8776,168,8719,8836,174,8658,216,8195,222,255,225,924,8660,8804,8201,8659,234,8222,931,197,732,8711,8212,8593,215,217,919,192,967,8476,710,240,8969,239,947,955,8596,200,190,8224,247,214,8465,8211,8230,236,376,8736,945,8260,208,8727,925,177,8226,185,178,179,193,162,8254,914,8869,916,8756,960,953,8709,235,8713,237,182,949,8472,338,252,8592,238,933,959,965,169,207,211,926,954,231,219,8745,956,353,8216,8712,918,8722,176,8743,964,163,164,8747,251,8971,8194,8629,249,8707,8773,952,8853,8240,194,982,203,934,205,34,218,927,8800,191,951,8217,253,929,8595,202,937,226,8764,966,962,175,977,199,186,8657,8704,946,402,961,181,233,969,183,915,8207,9001,9824,8839,254,246,8744,187,8706,167,8220,9829,963,243}; - for (int i=0; i - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -24.3 Character entity references for symbols, mathematical symbols, and Greek letters - -The character entity references in this section produce characters that may be represented by glyphs in the widely available Adobe Symbol font, including Greek characters, various bracketing symbols, and a selection of mathematical operators such as gradient, product, and summation symbols. - -To support these entities, user agents may support full [ISO10646] or use other means. Display of glyphs for these characters may be obtained by being able to display the relevant [ISO10646] characters or by other means, such as internally mapping the listed entities, numeric character references, and characters to the appropriate position in some font that contains the requisite glyphs. - -When to use Greek entities. This entity set contains all the letters used in modern Greek. However, it does not include Greek punctuation, precomposed accented characters nor the non-spacing accents (tonos, dialytika) required to compose them. There are no archaic letters, Coptic-unique letters, or precomposed letters for Polytonic Greek. The entities defined here are not intended for the representation of modern Greek text and would not be an efficient representation; rather, they are intended for occasional Greek letters used in technical and mathematical works. -24.3.1 The list of characters - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -24.4 Character entity references for markup-significant and internationalization characters - -The character entity references in this section are for escaping markup-significant characters (these are the same as those in HTML 2.0 and 3.2), for denoting spaces and dashes. Other characters in this section apply to internationalization issues such as the disambiguation of bidirectional text (see the section on bidirectional text for details). - -Entities have also been added for the remaining characters occurring in CP-1252 which do not occur in the HTMLlat1 or HTMLsymbol entity sets. These all occur in the 128 to 159 range within the CP-1252 charset. These entities permit the characters to be denoted in a platform-independent manner. - -To support these entities, user agents may support full [ISO10646] or use other means. Display of glyphs for these characters may be obtained by being able to display the relevant [ISO10646] characters or by other means, such as internally mapping the listed entities, numeric character references, and characters to the appropriate position in some font that contains the requisite glyphs. -24.4.1 The list of characters - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -""" - -codes={} -for line in text.split('\n'): - parts = line.split() - if len(parts)<3 or parts[0]!=' set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripReader(new StringReader(test), set); + Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -105,7 +107,7 @@ public class HTMLStripReaderTest extends TestCase { String gold = " < foo> = \u0393 bar \u0393 "; Set set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripReader(new StringReader(test), set); + Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -122,7 +124,7 @@ public class HTMLStripReaderTest extends TestCase { String gold = " < junk/> ! @ and ’ "; Set set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripReader(new StringReader(test), set); + Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -138,7 +140,7 @@ public class HTMLStripReaderTest extends TestCase { String test = "aaa bbb eeee ffff "; Set set = new HashSet(); set.add("reserved"); - Reader reader = new HTMLStripReader(new StringReader(test), set); + Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test)), set); StringBuilder builder = new StringBuilder(); int ch = 0; while ((ch = reader.read()) != -1){ @@ -195,7 +197,7 @@ public class HTMLStripReaderTest extends TestCase { "[aaaa://aaaaa.aaaaaaaa.aaa/aaaaaaa/aaaaaa/ aaaaaaa aaaaaa] aaaaaaaa aaaaaaaaaaaa aa aaaaaaaaaa, aaaaa aaaaaaaaa aaa [[aa aaaaaaa]] [[aaaa]]; aaaaaaaaaaa aaaaaaaa aaa [[aaa aa]] [[aaaa]]\n" + "\n" + "[[aaaaa aaaaaaaaaa]] aa ''[[aaa aaaaaaaaaaaa aa aaaaaa]]'' (aaaa) aaaa aaa aaaa [[zzzzzzz]] aa aaaaaaaa"; - Reader reader = new HTMLStripReader(new StringReader(test)); + Reader reader = new HTMLStripCharFilter(CharReader.get(new StringReader(test))); Reader noStrip = new StringReader(test); int ch = 0; int ch2 = 0; @@ -210,27 +212,27 @@ public class HTMLStripReaderTest extends TestCase { } public void testBufferOverflow() throws Exception { - StringBuilder testBuilder = new StringBuilder(HTMLStripReader.DEFAULT_READ_AHEAD + 50); + StringBuilder testBuilder = new StringBuilder(HTMLStripCharFilter.DEFAULT_READ_AHEAD + 50); testBuilder.append("ah "); - appendChars(testBuilder, HTMLStripReader.DEFAULT_READ_AHEAD + 500); + appendChars(testBuilder, HTMLStripCharFilter.DEFAULT_READ_AHEAD + 500); processBuffer(testBuilder.toString(), "Failed on pseudo proc. instr.");//processing instructions testBuilder.setLength(0); testBuilder.append("foo"); processBuffer(testBuilder.toString(), "Failed w/ comment"); testBuilder.setLength(0); testBuilder.append(""); processBuffer(testBuilder.toString(), "Failed with proc. instr."); testBuilder.setLength(0); testBuilder.append(""); processBuffer(testBuilder.toString(), "Failed on tag"); @@ -239,14 +241,14 @@ public class HTMLStripReaderTest extends TestCase { private void appendChars(StringBuilder testBuilder, int numChars) { int i1 = numChars / 2; for (int i = 0; i < i1; i++){ - testBuilder.append('a').append(' ');//tack on enough to go beyond the mark readahead limit, since makes HTMLStripReader think it is a processing instruction + testBuilder.append('a').append(' ');//tack on enough to go beyond the mark readahead limit, since makes HTMLStripCharFilter think it is a processing instruction } } private void processBuffer(String test, String assertMsg) throws IOException { System.out.println("-------------------processBuffer----------"); - Reader reader = new HTMLStripReader(new BufferedReader(new StringReader(test)));//force the use of BufferedReader + Reader reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -263,7 +265,7 @@ public class HTMLStripReaderTest extends TestCase { String test = " "; String gold = " "; - Reader reader = new HTMLStripReader(new BufferedReader(new StringReader(test)));//force the use of BufferedReader + Reader reader = new HTMLStripCharFilter(CharReader.get(new BufferedReader(new StringReader(test))));//force the use of BufferedReader int ch = 0; StringBuilder builder = new StringBuilder(); try { @@ -276,4 +278,4 @@ public class HTMLStripReaderTest extends TestCase { assertTrue(builder.toString() + " is not equal to " + gold + "", builder.toString().equals(gold) == true); } -} \ No newline at end of file +} diff --git a/src/webapp/web/admin/analysis.jsp b/src/webapp/web/admin/analysis.jsp index 6e20f29a334..d6ae6c50b88 100644 --- a/src/webapp/web/admin/analysis.jsp +++ b/src/webapp/web/admin/analysis.jsp @@ -174,7 +174,6 @@ <%! private static void doAnalyzer(JspWriter out, SchemaField field, String val, boolean queryAnalyser, boolean verbose, Set match) throws Exception { - CharStream reader = CharReader.get(new StringReader(val)); FieldType ft = field.getType(); Analyzer analyzer = queryAnalyser ? @@ -186,18 +185,18 @@ TokenFilterFactory[] filtfacs = tchain.getTokenFilterFactories(); if( cfiltfacs != null ){ + String source = val; for(CharFilterFactory cfiltfac : cfiltfacs ){ + CharStream reader = CharReader.get(new StringReader(source)); reader = cfiltfac.create(reader); if(verbose){ writeHeader(out, cfiltfac.getClass(), cfiltfac.getArgs()); - writeCharStream(out, reader); + source = writeCharStream(out, reader); } } } - // StringReader should support reset() - reader.reset(); - TokenStream tstream = tfac.create(reader); + TokenStream tstream = tfac.create(tchain.charStream(new StringReader(val))); List tokens = getTokens(tstream); if (verbose) { writeHeader(out, tfac.getClass(), tfac.getArgs()); @@ -223,7 +222,7 @@ } } else { - TokenStream tstream = analyzer.tokenStream(field.getName(),reader); + TokenStream tstream = analyzer.tokenStream(field.getName(),new StringReader(val)); List tokens = getTokens(tstream); if (verbose) { writeHeader(out, analyzer.getClass(), new HashMap()); @@ -468,7 +467,7 @@ out.println(""); } - static void writeCharStream(JspWriter out, CharStream input) throws IOException { + static String writeCharStream(JspWriter out, CharStream input) throws IOException { out.println(""); out.println(""); @@ -476,8 +475,6 @@ XML.escapeCharData("text",out); out.println(""); - // StringReader should support reset() - input.reset(); final int BUFFER_SIZE = 1024; char[] buf = new char[BUFFER_SIZE]; int len = 0; @@ -492,6 +489,7 @@ out.println(""); out.println("
"); + return sb.toString(); } %>