From 1a48e218d62b995acc0fe6b61f7cd2c45e3ac364 Mon Sep 17 00:00:00 2001 From: Otis Gospodnetic Date: Tue, 13 Mar 2007 00:30:13 +0000 Subject: [PATCH] - Committing forgotten classes for LUCENE-759 git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@517477 13f79535-47bb-0310-9956-ffa450edef68 --- .../analysis/ngram/EdgeNGramTokenFilter.java | 159 ++++++++++++++++++ .../analysis/ngram/NGramTokenFilter.java | 99 +++++++++++ 2 files changed, 258 insertions(+) create mode 100644 contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java create mode 100644 contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java diff --git a/contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java b/contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java new file mode 100644 index 00000000000..6d206c0e5a2 --- /dev/null +++ b/contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/EdgeNGramTokenFilter.java @@ -0,0 +1,159 @@ +package org.apache.lucene.analysis.ngram; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; + +import java.io.IOException; +import java.util.LinkedList; + +/** + * Tokenizes the given token into n-grams of given size(s). + * @author Otis Gospodnetic + */ +public class EdgeNGramTokenFilter extends TokenFilter { + public static final Side DEFAULT_SIDE = Side.FRONT; + public static final int DEFAULT_MAX_GRAM_SIZE = 1; + public static final int DEFAULT_MIN_GRAM_SIZE = 1; + + // Replace this with an enum when the Java 1.5 upgrade is made, the impl will be simplified + /** Specifies which side of the input the n-gram should be generated from */ + public static class Side { + private String label; + + /** Get the n-gram from the front of the input */ + public static Side FRONT = new Side("front"); + + /** Get the n-gram from the end of the input */ + public static Side BACK = new Side("back"); + + // Private ctor + private Side(String label) { this.label = label; } + + public String getLabel() { return label; } + + // Get the appropriate Side from a string + public static Side getSide(String sideName) { + if (FRONT.getLabel().equals(sideName)) { + return FRONT; + } + else if (BACK.getLabel().equals(sideName)) { + return BACK; + } + return null; + } + } + + private int minGram; + private int maxGram; + private Side side; + private LinkedList ngrams; + + protected EdgeNGramTokenFilter(TokenStream input) { + super(input); + this.ngrams = new LinkedList(); + } + + /** + * Creates EdgeNGramTokenFilter that can generate n-grams in the sizes of the given range + * + * @param input TokenStream holding the input to be tokenized + * @param side the {@link Side} from which to chop off an n-gram + * @param minGram the smallest n-gram to generate + * @param maxGram the largest n-gram to generate + */ + public EdgeNGramTokenFilter(TokenStream input, Side side, int minGram, int maxGram) { + super(input); + + if (side == null) { + throw new IllegalArgumentException("sideLabel must be either front or back"); + } + + if (minGram < 1) { + throw new IllegalArgumentException("minGram must be greater than zero"); + } + + if (minGram > maxGram) { + throw new IllegalArgumentException("minGram must not be greater than maxGram"); + } + + this.minGram = minGram; + this.maxGram = maxGram; + this.side = side; + this.ngrams = new LinkedList(); + } + + /** + * Creates EdgeNGramTokenFilter that can generate n-grams in the sizes of the given range + * + * @param input TokenStream holding the input to be tokenized + * @param sideLabel the name of the {@link Side} from which to chop off an n-gram + * @param minGram the smallest n-gram to generate + * @param maxGram the largest n-gram to generate + */ + public EdgeNGramTokenFilter(TokenStream input, String sideLabel, int minGram, int maxGram) { + this(input, Side.getSide(sideLabel), minGram, maxGram); + } + + /** Returns the next token in the stream, or null at EOS. */ + public final Token next() throws IOException { + if (ngrams.size() > 0) { + return (Token) ngrams.removeFirst(); + } + + Token token = input.next(); + if (token == null) { + return null; + } + + ngram(token); + if (ngrams.size() > 0) + return (Token) ngrams.removeFirst(); + else + return null; + } + + private void ngram(Token token) { + String inStr = token.termText(); + int inLen = inStr.length(); + int gramSize = minGram; + while (gramSize <= maxGram) { + // if the remaining input is too short, we can't generate any n-grams + if (gramSize > inLen) { + return; + } + + // if we have hit the end of our n-gram size range, quit + if (gramSize > maxGram) { + return; + } + + Token tok; + if (side == Side.FRONT) { + tok = new Token(inStr.substring(0, gramSize), 0, gramSize); + } + else { + tok = new Token(inStr.substring(inLen-gramSize), inLen-gramSize, inLen); + } + ngrams.add(tok); + gramSize++; + } + } +} diff --git a/contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java b/contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java new file mode 100644 index 00000000000..2441a8df885 --- /dev/null +++ b/contrib/analyzers/src/java/org/apache/lucene/analysis/ngram/NGramTokenFilter.java @@ -0,0 +1,99 @@ +package org.apache.lucene.analysis.ngram; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import org.apache.lucene.analysis.Token; +import org.apache.lucene.analysis.TokenFilter; +import org.apache.lucene.analysis.TokenStream; + +import java.io.IOException; +import java.util.LinkedList; + +/** + * Tokenizes the input into n-grams of the given size(s). + * @author Otis Gospodnetic + */ +public class NGramTokenFilter extends TokenFilter { + public static final int DEFAULT_MIN_NGRAM_SIZE = 1; + public static final int DEFAULT_MAX_NGRAM_SIZE = 2; + + private int minGram, maxGram; + private LinkedList ngrams; + + /** + * Creates NGramTokenFilter with given min and max n-grams. + * @param input TokenStream holding the input to be tokenized + * @param minGram the smallest n-gram to generate + * @param maxGram the largest n-gram to generate + */ + public NGramTokenFilter(TokenStream input, int minGram, int maxGram) { + super(input); + if (minGram < 1) { + throw new IllegalArgumentException("minGram must be greater than zero"); + } + if (minGram > maxGram) { + throw new IllegalArgumentException("minGram must not be greater than maxGram"); + } + this.minGram = minGram; + this.maxGram = maxGram; + this.ngrams = new LinkedList(); + } + + /** + * Creates NGramTokenFilter with default min and max n-grams. + * @param input TokenStream holding the input to be tokenized + */ + public NGramTokenFilter(TokenStream input) { + this(input, DEFAULT_MIN_NGRAM_SIZE, DEFAULT_MAX_NGRAM_SIZE); + } + + /** Returns the next token in the stream, or null at EOS. */ + public final Token next() throws IOException { + if (ngrams.size() > 0) { + return (Token) ngrams.removeFirst(); + } + + Token token = input.next(); + if (token == null) { + return null; + } + + ngram(token); + if (ngrams.size() > 0) + return (Token) ngrams.removeFirst(); + else + return null; + } + + private void ngram(Token token) { + String inStr = token.termText(); + int inLen = inStr.length(); + int gramSize = minGram; + while (gramSize <= maxGram) { + int pos = 0; // reset to beginning of string + while (pos+gramSize <= inLen) { // while there is input + String gram = inStr.substring(pos, pos+gramSize); + Token tok = new Token(gram, pos, pos+gramSize); +// tok.setPositionIncrement(pos); + ngrams.add(tok); + pos++; + } + gramSize++; // increase n-gram size + } + } +}