Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

svn:eolstyle=native 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1308638 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-04-03 00:59:11 +00:00
parent b230805157
commit 86c7878963
4 changed files with 289 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
dev-tools/idea/.idea/libraries/Morfologik_library.xml
View File

@ -1,10 +1,10 @@
<component name="libraryTable">
<library name="Morfologik library">
<CLASSES>
<root url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" />
</CLASSES>
<JAVADOC />
<SOURCES />
<jarDirectory url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" recursive="false" />
</library>
<component name="libraryTable">
<library name="Morfologik library">
<CLASSES>
<root url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" />
</CLASSES>
<JAVADOC />
<SOURCES />
<jarDirectory url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" recursive="false" />
</library>
</component>

162
solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java
View File

@ -1,81 +1,81 @@
package org.apache.solr.analysis;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Arrays;
import java.util.Locale;
import java.util.Map;
import morfologik.stemming.PolishStemmer.DICTIONARY;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.morfologik.MorfologikFilter;
/**
* Filter factory for {@link MorfologikFilter}.
* <pre class="prettyprint">
* &lt;fieldType name="text_polish" class="solr.TextField" positionIncrementGap="100"&gt;
* &lt;analyzer&gt;
* &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt;
* &lt;filter class="solr.MorfologikFilterFactory" dictionary="MORFOLOGIK" /&gt;
* &lt;/analyzer&gt;
* &lt;/fieldType&gt;</pre>
*
* <p>Any of Morfologik dictionaries can be used, these are at the moment:
* <code>MORFOLOGIK</code> (Morfologik's original dictionary),
* <code>MORFEUSZ</code> (Morfeusz-SIAT),
* <code>COMBINED</code> (both of the dictionaries above, combined).
*
* @see <a href="http://morfologik.blogspot.com/">Morfologik web site</a>
*/
public class MorfologikFilterFactory extends BaseTokenFilterFactory {
/** Dictionary. */
private DICTIONARY dictionary = DICTIONARY.MORFOLOGIK;
/** Schema attribute. */
public static final String DICTIONARY_SCHEMA_ATTRIBUTE = "dictionary";
/**
* {@inheritDoc}
*/
@Override
public TokenStream create(TokenStream ts) {
return new MorfologikFilter(ts, dictionary, luceneMatchVersion);
}
/**
* {@inheritDoc}
*/
@Override
public void init(Map<String,String> args) {
super.init(args);
String dictionaryName = args.get(DICTIONARY_SCHEMA_ATTRIBUTE);
if (dictionaryName != null && !dictionaryName.isEmpty()) {
try {
DICTIONARY dictionary = DICTIONARY.valueOf(dictionaryName.toUpperCase(Locale.ENGLISH));
assert dictionary != null;
this.dictionary = dictionary;
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("The " + DICTIONARY_SCHEMA_ATTRIBUTE + " attribute accepts the "
+ "following constants: " + Arrays.toString(DICTIONARY.values()) + ", this value is invalid: "
+ dictionaryName);
}
}
}
}
package org.apache.solr.analysis;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.Arrays;
import java.util.Locale;
import java.util.Map;
import morfologik.stemming.PolishStemmer.DICTIONARY;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.morfologik.MorfologikFilter;
/**
* Filter factory for {@link MorfologikFilter}.
* <pre class="prettyprint">
* &lt;fieldType name="text_polish" class="solr.TextField" positionIncrementGap="100"&gt;
* &lt;analyzer&gt;
* &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt;
* &lt;filter class="solr.MorfologikFilterFactory" dictionary="MORFOLOGIK" /&gt;
* &lt;/analyzer&gt;
* &lt;/fieldType&gt;</pre>
*
* <p>Any of Morfologik dictionaries can be used, these are at the moment:
* <code>MORFOLOGIK</code> (Morfologik's original dictionary),
* <code>MORFEUSZ</code> (Morfeusz-SIAT),
* <code>COMBINED</code> (both of the dictionaries above, combined).
*
* @see <a href="http://morfologik.blogspot.com/">Morfologik web site</a>
*/
public class MorfologikFilterFactory extends BaseTokenFilterFactory {
/** Dictionary. */
private DICTIONARY dictionary = DICTIONARY.MORFOLOGIK;
/** Schema attribute. */
public static final String DICTIONARY_SCHEMA_ATTRIBUTE = "dictionary";
/**
* {@inheritDoc}
*/
@Override
public TokenStream create(TokenStream ts) {
return new MorfologikFilter(ts, dictionary, luceneMatchVersion);
}
/**
* {@inheritDoc}
*/
@Override
public void init(Map<String,String> args) {
super.init(args);
String dictionaryName = args.get(DICTIONARY_SCHEMA_ATTRIBUTE);
if (dictionaryName != null && !dictionaryName.isEmpty()) {
try {
DICTIONARY dictionary = DICTIONARY.valueOf(dictionaryName.toUpperCase(Locale.ENGLISH));
assert dictionary != null;
this.dictionary = dictionary;
} catch (IllegalArgumentException e) {
throw new IllegalArgumentException("The " + DICTIONARY_SCHEMA_ATTRIBUTE + " attribute accepts the "
+ "following constants: " + Arrays.toString(DICTIONARY.values()) + ", this value is invalid: "
+ dictionaryName);
}
}
}
}

90
solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java
View File

@ -1,45 +1,45 @@
package org.apache.solr.analysis;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.schema.IndexSchema;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Test for {@link MorfologikFilterFactory}.
*/
public class TestMorfologikFilterFactory extends BaseTokenTestCase {
public void testCreateDictionary() throws Exception {
StringReader reader = new StringReader("rowery bilety");
Map<String,String> initParams = new HashMap<String,String>();
initParams.put(IndexSchema.LUCENE_MATCH_VERSION_PARAM,
DEFAULT_VERSION.toString());
initParams.put(MorfologikFilterFactory.DICTIONARY_SCHEMA_ATTRIBUTE,
"morfologik");
MorfologikFilterFactory factory = new MorfologikFilterFactory();
factory.init(initParams);
TokenStream ts = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION,
reader));
assertTokenStreamContents(ts, new String[] {"rower", "bilet"});
}
}
package org.apache.solr.analysis;
import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.schema.IndexSchema;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/**
* Test for {@link MorfologikFilterFactory}.
*/
public class TestMorfologikFilterFactory extends BaseTokenTestCase {
public void testCreateDictionary() throws Exception {
StringReader reader = new StringReader("rowery bilety");
Map<String,String> initParams = new HashMap<String,String>();
initParams.put(IndexSchema.LUCENE_MATCH_VERSION_PARAM,
DEFAULT_VERSION.toString());
initParams.put(MorfologikFilterFactory.DICTIONARY_SCHEMA_ATTRIBUTE,
"morfologik");
MorfologikFilterFactory factory = new MorfologikFilterFactory();
factory.init(initParams);
TokenStream ts = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION,
reader));
assertTokenStreamContents(ts, new String[] {"rower", "bilet"});
}
}

308
solr/solrj/src/java/org/apache/noggit/ObjectBuilder.java
View File

@ -1,154 +1,154 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.noggit;
import java.util.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
/**
* @author yonik
* @version $Id$
*/
public class ObjectBuilder {
public static Object fromJSON(String json) throws IOException {
JSONParser p = new JSONParser(json);
return getVal(p);
}
public static Object getVal(JSONParser parser) throws IOException {
return new ObjectBuilder(parser).getVal();
}
final JSONParser parser;
public ObjectBuilder(JSONParser parser) throws IOException {
this.parser = parser;
if (parser.lastEvent()==0) parser.nextEvent();
}
public Object getVal() throws IOException {
int ev = parser.lastEvent();
switch(ev) {
case JSONParser.STRING: return getString();
case JSONParser.LONG: return getLong();
case JSONParser.NUMBER: return getNumber();
case JSONParser.BIGNUMBER: return getBigNumber();
case JSONParser.BOOLEAN: return getBoolean();
case JSONParser.NULL: return getNull();
case JSONParser.OBJECT_START: return getObject();
case JSONParser.OBJECT_END: return null; // OR ERROR?
case JSONParser.ARRAY_START: return getArray();
case JSONParser.ARRAY_END: return null; // OR ERROR?
case JSONParser.EOF: return null; // OR ERROR?
default: return null; // OR ERROR?
}
}
public Object getString() throws IOException {
return parser.getString();
}
public Object getLong() throws IOException {
return Long.valueOf(parser.getLong());
}
public Object getNumber() throws IOException {
CharArr num = parser.getNumberChars();
String numstr = num.toString();
double d = Double.parseDouble(numstr);
if (!Double.isInfinite(d)) return Double.valueOf(d);
// TODO: use more efficient constructor in Java5
return new BigDecimal(numstr);
}
public Object getBigNumber() throws IOException {
CharArr num = parser.getNumberChars();
String numstr = num.toString();
for(int ch; (ch=num.read())!=-1;) {
if (ch=='.' || ch=='e' || ch=='E') return new BigDecimal(numstr);
}
return new BigInteger(numstr);
}
public Object getBoolean() throws IOException {
return parser.getBoolean();
}
public Object getNull() throws IOException {
parser.getNull();
return null;
}
public Object newObject() throws IOException {
return new LinkedHashMap();
}
public Object getKey() throws IOException {
return parser.getString();
}
public void addKeyVal(Object map, Object key, Object val) throws IOException {
Object prev = ((Map)map).put(key,val);
// TODO: test for repeated value?
}
public Object objectEnd(Object obj) {
return obj;
}
public Object getObject() throws IOException {
Object m = newObject();
for(;;) {
int ev = parser.nextEvent();
if (ev==JSONParser.OBJECT_END) return objectEnd(m);
Object key = getKey();
ev = parser.nextEvent();
Object val = getVal();
addKeyVal(m, key, val);
}
}
public Object newArray() {
return new ArrayList();
}
public void addArrayVal(Object arr, Object val) throws IOException {
((List)arr).add(val);
}
public Object endArray(Object arr) {
return arr;
}
public Object getArray() throws IOException {
Object arr = newArray();
for(;;) {
int ev = parser.nextEvent();
if (ev==JSONParser.ARRAY_END) return endArray(arr);
Object val = getVal();
addArrayVal(arr, val);
}
}
}
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.noggit;
import java.util.*;
import java.io.IOException;
import java.math.BigDecimal;
import java.math.BigInteger;
/**
* @author yonik
* @version $Id$
*/
public class ObjectBuilder {
public static Object fromJSON(String json) throws IOException {
JSONParser p = new JSONParser(json);
return getVal(p);
}
public static Object getVal(JSONParser parser) throws IOException {
return new ObjectBuilder(parser).getVal();
}
final JSONParser parser;
public ObjectBuilder(JSONParser parser) throws IOException {
this.parser = parser;
if (parser.lastEvent()==0) parser.nextEvent();
}
public Object getVal() throws IOException {
int ev = parser.lastEvent();
switch(ev) {
case JSONParser.STRING: return getString();
case JSONParser.LONG: return getLong();
case JSONParser.NUMBER: return getNumber();
case JSONParser.BIGNUMBER: return getBigNumber();
case JSONParser.BOOLEAN: return getBoolean();
case JSONParser.NULL: return getNull();
case JSONParser.OBJECT_START: return getObject();
case JSONParser.OBJECT_END: return null; // OR ERROR?
case JSONParser.ARRAY_START: return getArray();
case JSONParser.ARRAY_END: return null; // OR ERROR?
case JSONParser.EOF: return null; // OR ERROR?
default: return null; // OR ERROR?
}
}
public Object getString() throws IOException {
return parser.getString();
}
public Object getLong() throws IOException {
return Long.valueOf(parser.getLong());
}
public Object getNumber() throws IOException {
CharArr num = parser.getNumberChars();
String numstr = num.toString();
double d = Double.parseDouble(numstr);
if (!Double.isInfinite(d)) return Double.valueOf(d);
// TODO: use more efficient constructor in Java5
return new BigDecimal(numstr);
}
public Object getBigNumber() throws IOException {
CharArr num = parser.getNumberChars();
String numstr = num.toString();
for(int ch; (ch=num.read())!=-1;) {
if (ch=='.' || ch=='e' || ch=='E') return new BigDecimal(numstr);
}
return new BigInteger(numstr);
}
public Object getBoolean() throws IOException {
return parser.getBoolean();
}
public Object getNull() throws IOException {
parser.getNull();
return null;
}
public Object newObject() throws IOException {
return new LinkedHashMap();
}
public Object getKey() throws IOException {
return parser.getString();
}
public void addKeyVal(Object map, Object key, Object val) throws IOException {
Object prev = ((Map)map).put(key,val);
// TODO: test for repeated value?
}
public Object objectEnd(Object obj) {
return obj;
}
public Object getObject() throws IOException {
Object m = newObject();
for(;;) {
int ev = parser.nextEvent();
if (ev==JSONParser.OBJECT_END) return objectEnd(m);
Object key = getKey();
ev = parser.nextEvent();
Object val = getVal();
addKeyVal(m, key, val);
}
}
public Object newArray() {
return new ArrayList();
}
public void addArrayVal(Object arr, Object val) throws IOException {
((List)arr).add(val);
}
public Object endArray(Object arr) {
return arr;
}
public Object getArray() throws IOException {
Object arr = newArray();
for(;;) {
int ev = parser.nextEvent();
if (ev==JSONParser.ARRAY_END) return endArray(arr);
Object val = getVal();
addArrayVal(arr, val);
}
}
}