mirror of https://github.com/apache/lucene.git
tentative add: Various fulltext analysis utilities avoiding redundant code in several
* classes. git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@351895 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Wolfgang Hoschek
parent
860733f32e
commit
a155416b4d
|
|
@ -0,0 +1,382 @@
|
|||
package org.apache.lucene.index.memory;
|
||||
|
||||
/**
|
||||
* Copyright 2005 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.PrintStream;
|
||||
import java.io.Reader;
|
||||
import java.io.StringReader;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.HashMap;
|
||||
import java.util.Map;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
import org.apache.lucene.analysis.Analyzer;
|
||||
import org.apache.lucene.analysis.PorterStemFilter;
|
||||
import org.apache.lucene.analysis.Token;
|
||||
import org.apache.lucene.analysis.TokenFilter;
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
|
||||
/**
|
||||
* Various fulltext analysis utilities avoiding redundant code in several
|
||||
* classes.
|
||||
*
|
||||
* @author whoschek.AT.lbl.DOT.gov
|
||||
*/
|
||||
public class AnalyzerUtil {
|
||||
|
||||
private AnalyzerUtil() {};
|
||||
|
||||
/**
|
||||
* Returns a simple analyzer wrapper that logs all tokens produced by the
|
||||
* underlying child analyzer to the given log stream (typically System.err);
|
||||
* Otherwise behaves exactly like the child analyzer, delivering the very
|
||||
* same tokens; useful for debugging purposes on custom indexing and/or
|
||||
* querying.
|
||||
*
|
||||
* @param child
|
||||
* the underlying child analyzer
|
||||
* @param log
|
||||
* the print stream to log to (typically System.err)
|
||||
* @param logName
|
||||
* a name for this logger (typically "log" or similar)
|
||||
* @return a logging analyzer
|
||||
*/
|
||||
public static Analyzer getLoggingAnalyzer(final Analyzer child,
|
||||
final PrintStream log, final String logName) {
|
||||
|
||||
if (child == null)
|
||||
throw new IllegalArgumentException("child analyzer must not be null");
|
||||
if (log == null)
|
||||
throw new IllegalArgumentException("logStream must not be null");
|
||||
|
||||
return new Analyzer() {
|
||||
public TokenStream tokenStream(final String fieldName, Reader reader) {
|
||||
return new TokenFilter(child.tokenStream(fieldName, reader)) {
|
||||
private int position = -1;
|
||||
|
||||
public Token next() throws IOException {
|
||||
Token token = input.next(); // from filter super class
|
||||
log.println(toString(token));
|
||||
return token;
|
||||
}
|
||||
|
||||
private String toString(Token token) {
|
||||
if (token == null) return "[" + logName + ":EOS:" + fieldName + "]\n";
|
||||
|
||||
position += token.getPositionIncrement();
|
||||
return "[" + logName + ":" + position + ":" + fieldName + ":"
|
||||
+ token.termText() + ":" + token.startOffset()
|
||||
+ "-" + token.endOffset() + ":" + token.type()
|
||||
+ "]";
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns an analyzer wrapper that returns at most the first
|
||||
* <code>maxTokens</code> tokens from the underlying child analyzer,
|
||||
* ignoring all remaining tokens.
|
||||
*
|
||||
* @param child
|
||||
* the underlying child analyzer
|
||||
* @param maxTokens
|
||||
* the maximum number of tokens to return from the underlying
|
||||
* analyzer (a value of Integer.MAX_VALUE indicates unlimited)
|
||||
* @return an analyzer wrapper
|
||||
*/
|
||||
public static Analyzer getMaxTokenAnalyzer(
|
||||
final Analyzer child, final int maxTokens, final int maxChars) {
|
||||
|
||||
if (child == null)
|
||||
throw new IllegalArgumentException("child analyzer must not be null");
|
||||
if (maxTokens < 0)
|
||||
throw new IllegalArgumentException("maxTokens must not be negative");
|
||||
if (maxTokens == Integer.MAX_VALUE)
|
||||
return child; // no need to wrap
|
||||
|
||||
return new Analyzer() {
|
||||
public TokenStream tokenStream(String fieldName, Reader reader) {
|
||||
return new TokenFilter(child.tokenStream(fieldName, reader)) {
|
||||
private int todo = maxTokens;
|
||||
|
||||
public Token next() throws IOException {
|
||||
return --todo >= 0 ? input.next() : null;
|
||||
}
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns an English stemming analyzer that stems tokens from the
|
||||
* underlying child analyzer according to the Porter stemming algorithm. The
|
||||
* child analyzer must deliver tokens in lower case for the stemmer to work
|
||||
* properly.
|
||||
* <p>
|
||||
* Background: Stemming reduces token terms to their linguistic root form
|
||||
* e.g. reduces "fishing" and "fishes" to "fish", "family" and "families" to
|
||||
* "famili", as well as "complete" and "completion" to "complet". Note that
|
||||
* the root form is not necessarily a meaningful word in itself, and that
|
||||
* this is not a bug but rather a feature, if you lean back and think about
|
||||
* fuzzy word matching for a bit.
|
||||
* <p>
|
||||
* See the Lucene contrib packages for stemmers (and stop words) for German,
|
||||
* Russian and many more languages.
|
||||
*
|
||||
* @param child
|
||||
* the underlying child analyzer
|
||||
* @return an analyzer wrapper
|
||||
*/
|
||||
public static Analyzer getPorterStemmerAnalyzer(final Analyzer child) {
|
||||
|
||||
if (child == null)
|
||||
throw new IllegalArgumentException("child analyzer must not be null");
|
||||
|
||||
return new Analyzer() {
|
||||
public TokenStream tokenStream(String fieldName, Reader reader) {
|
||||
return new PorterStemFilter(
|
||||
child.tokenStream(fieldName, reader));
|
||||
// /* PorterStemFilter and SnowballFilter have the same behaviour,
|
||||
// but PorterStemFilter is much faster. */
|
||||
// return new org.apache.lucene.analysis.snowball.SnowballFilter(
|
||||
// child.tokenStream(fieldName, reader), "English");
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns an analyzer wrapper that wraps the underlying child analyzer's
|
||||
* token stream into a {@link SynonymTokenFilter}.
|
||||
*
|
||||
* @param child
|
||||
* the underlying child analyzer
|
||||
* @param synonyms
|
||||
* the map used to extract synonyms for terms
|
||||
* @param maxSynonyms
|
||||
* the maximum number of synonym tokens to return per underlying
|
||||
* token word (a value of Integer.MAX_VALUE indicates unlimited)
|
||||
*/
|
||||
public static Analyzer getSynonymAnalyzer(final Analyzer child,
|
||||
final SynonymMap synonyms, final int maxSynonyms) {
|
||||
|
||||
if (child == null)
|
||||
throw new IllegalArgumentException("child analyzer must not be null");
|
||||
if (synonyms == null)
|
||||
throw new IllegalArgumentException("synonyms must not be null");
|
||||
if (maxSynonyms < 0)
|
||||
throw new IllegalArgumentException("maxSynonyms must not be negative");
|
||||
if (maxSynonyms == 0)
|
||||
return child; // no need to wrap
|
||||
|
||||
return new Analyzer() {
|
||||
public TokenStream tokenStream(String fieldName, Reader reader) {
|
||||
return new SynonymTokenFilter(
|
||||
child.tokenStream(fieldName, reader), synonyms, maxSynonyms);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Returns (frequency:text) pairs for the top N distinct terms (aka words),
|
||||
* sorted descending by frequency (and ascending by term, if tied).
|
||||
* <p>
|
||||
* Example XQuery:
|
||||
* <pre>
|
||||
* declare namespace util = "java:org.apache.lucene.index.memory.AnalyzerUtil";
|
||||
* declare namespace analyzer = "java:org.apache.lucene.index.memory.PatternAnalyzer";
|
||||
*
|
||||
* for $pair in util:get-most-frequent-terms(
|
||||
* analyzer:EXTENDED_ANALYZER(), doc("samples/shakespeare/othello.xml"), 10)
|
||||
* return <word word="{substring-after($pair, ':')}" frequency="{substring-before($pair, ':')}"/>
|
||||
* </pre>
|
||||
*
|
||||
* @param analyzer
|
||||
* the analyzer to use for splitting text into terms (aka words)
|
||||
* @param text
|
||||
* the text to analyze
|
||||
* @param limit
|
||||
* the maximum number of pairs to return; zero indicates
|
||||
* "as many as possible".
|
||||
* @return an array of (frequency:text) pairs in the form of (freq0:text0,
|
||||
* freq1:text1, ..., freqN:textN). Each pair is a single string
|
||||
* separated by a ':' delimiter.
|
||||
*/
|
||||
public static String[] getMostFrequentTerms(Analyzer analyzer, String text, int limit) {
|
||||
if (analyzer == null)
|
||||
throw new IllegalArgumentException("analyzer must not be null");
|
||||
if (text == null)
|
||||
throw new IllegalArgumentException("text must not be null");
|
||||
if (limit <= 0) limit = Integer.MAX_VALUE;
|
||||
|
||||
// compute frequencies of distinct terms
|
||||
HashMap map = new HashMap();
|
||||
TokenStream stream = analyzer.tokenStream("", new StringReader(text));
|
||||
try {
|
||||
Token token;
|
||||
while ((token = stream.next()) != null) {
|
||||
MutableInteger freq = (MutableInteger) map.get(token.termText());
|
||||
if (freq == null) {
|
||||
freq = new MutableInteger(1);
|
||||
map.put(token.termText(), freq);
|
||||
} else {
|
||||
freq.setValue(freq.intValue() + 1);
|
||||
}
|
||||
}
|
||||
} catch (IOException e) {
|
||||
throw new RuntimeException(e);
|
||||
} finally {
|
||||
try {
|
||||
stream.close();
|
||||
} catch (IOException e2) {
|
||||
throw new RuntimeException(e2);
|
||||
}
|
||||
}
|
||||
|
||||
// sort by frequency, text
|
||||
Map.Entry[] entries = new Map.Entry[map.size()];
|
||||
map.entrySet().toArray(entries);
|
||||
Arrays.sort(entries, new Comparator() {
|
||||
public int compare(Object o1, Object o2) {
|
||||
Map.Entry e1 = (Map.Entry) o1;
|
||||
Map.Entry e2 = (Map.Entry) o2;
|
||||
int f1 = ((MutableInteger) e1.getValue()).intValue();
|
||||
int f2 = ((MutableInteger) e2.getValue()).intValue();
|
||||
if (f2 - f1 != 0) return f2 - f1;
|
||||
String s1 = (String) e1.getKey();
|
||||
String s2 = (String) e2.getKey();
|
||||
return s1.compareTo(s2);
|
||||
}
|
||||
});
|
||||
|
||||
// return top N entries
|
||||
int size = Math.min(limit, entries.length);
|
||||
String[] pairs = new String[size];
|
||||
for (int i=0; i < size; i++) {
|
||||
pairs[i] = entries[i].getValue() + ":" + entries[i].getKey();
|
||||
}
|
||||
return pairs;
|
||||
}
|
||||
|
||||
private static final class MutableInteger {
|
||||
private int value;
|
||||
public MutableInteger(int value) { this.value = value; }
|
||||
public int intValue() { return value; }
|
||||
public void setValue(int value) { this.value = value; }
|
||||
public String toString() { return String.valueOf(value); }
|
||||
};
|
||||
|
||||
|
||||
|
||||
// TODO: could use a more general i18n approach ala http://icu.sourceforge.net/docs/papers/text_boundary_analysis_in_java/
|
||||
/** (Line terminator followed by zero or more whitespace) two or more times */
|
||||
private static final Pattern PARAGRAPHS = Pattern.compile("([\\r\\n\\u0085\\u2028\\u2029][ \\t\\x0B\\f]*){2,}");
|
||||
|
||||
/**
|
||||
* Returns at most the first N paragraphs of the given text. Delimiting
|
||||
* characters are excluded from the results. Each returned paragraph is
|
||||
* whitespace-trimmed via String.trim(), potentially an empty string.
|
||||
*
|
||||
* @param text
|
||||
* the text to tokenize into paragraphs
|
||||
* @param limit
|
||||
* the maximum number of paragraphs to return; zero indicates "as
|
||||
* many as possible".
|
||||
* @return the first N paragraphs
|
||||
*/
|
||||
public static String[] getParagraphs(String text, int limit) {
|
||||
return tokenize(PARAGRAPHS, text, limit);
|
||||
}
|
||||
|
||||
private static String[] tokenize(Pattern pattern, String text, int limit) {
|
||||
String[] tokens = pattern.split(text, limit);
|
||||
for (int i=tokens.length; --i >= 0; ) tokens[i] = tokens[i].trim();
|
||||
return tokens;
|
||||
}
|
||||
|
||||
|
||||
// TODO: don't split on floating point numbers, e.g. 3.1415 (digit before or after '.')
|
||||
/** Divides text into sentences; Includes inverted spanish exclamation and question mark */
|
||||
private static final Pattern SENTENCES = Pattern.compile("[!\\.\\?\\xA1\\xBF]+");
|
||||
|
||||
/**
|
||||
* Returns at most the first N sentences of the given text. Delimiting
|
||||
* characters are excluded from the results. Each returned sentence is
|
||||
* whitespace-trimmed via String.trim(), potentially an empty string.
|
||||
*
|
||||
* @param text
|
||||
* the text to tokenize into sentences
|
||||
* @param limit
|
||||
* the maximum number of sentences to return; zero indicates "as
|
||||
* many as possible".
|
||||
* @return the first N sentences
|
||||
*/
|
||||
public static String[] getSentences(String text, int limit) {
|
||||
// return tokenize(SENTENCES, text, limit); // equivalent but slower
|
||||
int len = text.length();
|
||||
if (len == 0) return new String[] { text };
|
||||
if (limit <= 0) limit = Integer.MAX_VALUE;
|
||||
|
||||
// average sentence length heuristic
|
||||
String[] tokens = new String[Math.min(limit, 1 + len/40)];
|
||||
int size = 0;
|
||||
int i = 0;
|
||||
|
||||
while (i < len && size < limit) {
|
||||
|
||||
// scan to end of current sentence
|
||||
int start = i;
|
||||
while (i < len && !isSentenceSeparator(text.charAt(i))) i++;
|
||||
|
||||
if (size == tokens.length) { // grow array
|
||||
String[] tmp = new String[tokens.length << 1];
|
||||
System.arraycopy(tokens, 0, tmp, 0, size);
|
||||
tokens = tmp;
|
||||
}
|
||||
// add sentence (potentially empty)
|
||||
tokens[size++] = text.substring(start, i).trim();
|
||||
|
||||
// scan to beginning of next sentence
|
||||
while (i < len && isSentenceSeparator(text.charAt(i))) i++;
|
||||
}
|
||||
|
||||
if (size == tokens.length) return tokens;
|
||||
String[] results = new String[size];
|
||||
System.arraycopy(tokens, 0, results, 0, size);
|
||||
return results;
|
||||
}
|
||||
|
||||
private static boolean isSentenceSeparator(char c) {
|
||||
// regex [!\\.\\?\\xA1\\xBF]
|
||||
switch (c) {
|
||||
case '!': return true;
|
||||
case '.': return true;
|
||||
case '?': return true;
|
||||
case 0xA1: return true; // spanish inverted exclamation mark
|
||||
case 0xBF: return true; // spanish inverted question mark
|
||||
default: return false;
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
Loading…
Reference in New Issue