Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

svn:eolstyle=native 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1308638 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2012-04-03 00:59:11 +00:00
parent b230805157
commit 86c7878963
4 changed files with 289 additions and 289 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
dev-tools/idea/.idea/libraries/Morfologik_library.xml
View File

@ -1,10 +1,10 @@
<component name="libraryTable"> <component name="libraryTable">
<library name="Morfologik library"> <library name="Morfologik library">
<CLASSES> <CLASSES>
<root url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" /> <root url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" />
</CLASSES> </CLASSES>
<JAVADOC /> <JAVADOC />
<SOURCES /> <SOURCES />
<jarDirectory url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" recursive="false" /> <jarDirectory url="file://$PROJECT_DIR$/modules/analysis/morfologik/lib" recursive="false" />
</library> </library>
</component> </component>

162
solr/contrib/analysis-extras/src/java/org/apache/solr/analysis/MorfologikFilterFactory.java
View File

@ -1,81 +1,81 @@
package org.apache.solr.analysis; package org.apache.solr.analysis;
/** /**
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. * this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0 * The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with * (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at * the License. You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
import java.util.Arrays; import java.util.Arrays;
import java.util.Locale; import java.util.Locale;
import java.util.Map; import java.util.Map;
import morfologik.stemming.PolishStemmer.DICTIONARY; import morfologik.stemming.PolishStemmer.DICTIONARY;
import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.morfologik.MorfologikFilter; import org.apache.lucene.analysis.morfologik.MorfologikFilter;
/** /**
* Filter factory for {@link MorfologikFilter}. * Filter factory for {@link MorfologikFilter}.
* <pre class="prettyprint"> * <pre class="prettyprint">
* &lt;fieldType name="text_polish" class="solr.TextField" positionIncrementGap="100"&gt; * &lt;fieldType name="text_polish" class="solr.TextField" positionIncrementGap="100"&gt;
* &lt;analyzer&gt; * &lt;analyzer&gt;
* &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt; * &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt;
* &lt;filter class="solr.MorfologikFilterFactory" dictionary="MORFOLOGIK" /&gt; * &lt;filter class="solr.MorfologikFilterFactory" dictionary="MORFOLOGIK" /&gt;
* &lt;/analyzer&gt; * &lt;/analyzer&gt;
* &lt;/fieldType&gt;</pre> * &lt;/fieldType&gt;</pre>
* *
* <p>Any of Morfologik dictionaries can be used, these are at the moment: * <p>Any of Morfologik dictionaries can be used, these are at the moment:
* <code>MORFOLOGIK</code> (Morfologik's original dictionary), * <code>MORFOLOGIK</code> (Morfologik's original dictionary),
* <code>MORFEUSZ</code> (Morfeusz-SIAT), * <code>MORFEUSZ</code> (Morfeusz-SIAT),
* <code>COMBINED</code> (both of the dictionaries above, combined). * <code>COMBINED</code> (both of the dictionaries above, combined).
* *
* @see <a href="http://morfologik.blogspot.com/">Morfologik web site</a> * @see <a href="http://morfologik.blogspot.com/">Morfologik web site</a>
*/ */
public class MorfologikFilterFactory extends BaseTokenFilterFactory { public class MorfologikFilterFactory extends BaseTokenFilterFactory {
/** Dictionary. */ /** Dictionary. */
private DICTIONARY dictionary = DICTIONARY.MORFOLOGIK; private DICTIONARY dictionary = DICTIONARY.MORFOLOGIK;
/** Schema attribute. */ /** Schema attribute. */
public static final String DICTIONARY_SCHEMA_ATTRIBUTE = "dictionary"; public static final String DICTIONARY_SCHEMA_ATTRIBUTE = "dictionary";
/** /**
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public TokenStream create(TokenStream ts) { public TokenStream create(TokenStream ts) {
return new MorfologikFilter(ts, dictionary, luceneMatchVersion); return new MorfologikFilter(ts, dictionary, luceneMatchVersion);
} }
/** /**
* {@inheritDoc} * {@inheritDoc}
*/ */
@Override @Override
public void init(Map<String,String> args) { public void init(Map<String,String> args) {
super.init(args); super.init(args);
String dictionaryName = args.get(DICTIONARY_SCHEMA_ATTRIBUTE); String dictionaryName = args.get(DICTIONARY_SCHEMA_ATTRIBUTE);
if (dictionaryName != null && !dictionaryName.isEmpty()) { if (dictionaryName != null && !dictionaryName.isEmpty()) {
try { try {
DICTIONARY dictionary = DICTIONARY.valueOf(dictionaryName.toUpperCase(Locale.ENGLISH)); DICTIONARY dictionary = DICTIONARY.valueOf(dictionaryName.toUpperCase(Locale.ENGLISH));
assert dictionary != null; assert dictionary != null;
this.dictionary = dictionary; this.dictionary = dictionary;
} catch (IllegalArgumentException e) { } catch (IllegalArgumentException e) {
throw new IllegalArgumentException("The " + DICTIONARY_SCHEMA_ATTRIBUTE + " attribute accepts the " throw new IllegalArgumentException("The " + DICTIONARY_SCHEMA_ATTRIBUTE + " attribute accepts the "
+ "following constants: " + Arrays.toString(DICTIONARY.values()) + ", this value is invalid: " + "following constants: " + Arrays.toString(DICTIONARY.values()) + ", this value is invalid: "
+ dictionaryName); + dictionaryName);
} }
} }
} }
} }

90
solr/contrib/analysis-extras/src/test/org/apache/solr/analysis/TestMorfologikFilterFactory.java
View File

@ -1,45 +1,45 @@
package org.apache.solr.analysis; package org.apache.solr.analysis;
import java.io.StringReader; import java.io.StringReader;
import java.util.HashMap; import java.util.HashMap;
import java.util.Map; import java.util.Map;
import org.apache.lucene.analysis.TokenStream; import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.core.WhitespaceTokenizer; import org.apache.lucene.analysis.core.WhitespaceTokenizer;
import org.apache.solr.schema.IndexSchema; import org.apache.solr.schema.IndexSchema;
/** /**
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. * this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0 * The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with * (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at * the License. You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
/** /**
* Test for {@link MorfologikFilterFactory}. * Test for {@link MorfologikFilterFactory}.
*/ */
public class TestMorfologikFilterFactory extends BaseTokenTestCase { public class TestMorfologikFilterFactory extends BaseTokenTestCase {
public void testCreateDictionary() throws Exception { public void testCreateDictionary() throws Exception {
StringReader reader = new StringReader("rowery bilety"); StringReader reader = new StringReader("rowery bilety");
Map<String,String> initParams = new HashMap<String,String>(); Map<String,String> initParams = new HashMap<String,String>();
initParams.put(IndexSchema.LUCENE_MATCH_VERSION_PARAM, initParams.put(IndexSchema.LUCENE_MATCH_VERSION_PARAM,
DEFAULT_VERSION.toString()); DEFAULT_VERSION.toString());
initParams.put(MorfologikFilterFactory.DICTIONARY_SCHEMA_ATTRIBUTE, initParams.put(MorfologikFilterFactory.DICTIONARY_SCHEMA_ATTRIBUTE,
"morfologik"); "morfologik");
MorfologikFilterFactory factory = new MorfologikFilterFactory(); MorfologikFilterFactory factory = new MorfologikFilterFactory();
factory.init(initParams); factory.init(initParams);
TokenStream ts = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION, TokenStream ts = factory.create(new WhitespaceTokenizer(DEFAULT_VERSION,
reader)); reader));
assertTokenStreamContents(ts, new String[] {"rower", "bilet"}); assertTokenStreamContents(ts, new String[] {"rower", "bilet"});
} }
} }

308
solr/solrj/src/java/org/apache/noggit/ObjectBuilder.java
View File

@ -1,154 +1,154 @@
/** /**
* Licensed to the Apache Software Foundation (ASF) under one or more * Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with * contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership. * this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0 * The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with * (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at * the License. You may obtain a copy of the License at
* *
* http://www.apache.org/licenses/LICENSE-2.0 * http://www.apache.org/licenses/LICENSE-2.0
* *
* Unless required by applicable law or agreed to in writing, software * Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, * distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and * See the License for the specific language governing permissions and
* limitations under the License. * limitations under the License.
*/ */
package org.apache.noggit; package org.apache.noggit;
import java.util.*; import java.util.*;
import java.io.IOException; import java.io.IOException;
import java.math.BigDecimal; import java.math.BigDecimal;
import java.math.BigInteger; import java.math.BigInteger;
/** /**
* @author yonik * @author yonik
* @version $Id$ * @version $Id$
*/ */
public class ObjectBuilder { public class ObjectBuilder {
public static Object fromJSON(String json) throws IOException { public static Object fromJSON(String json) throws IOException {
JSONParser p = new JSONParser(json); JSONParser p = new JSONParser(json);
return getVal(p); return getVal(p);
} }
public static Object getVal(JSONParser parser) throws IOException { public static Object getVal(JSONParser parser) throws IOException {
return new ObjectBuilder(parser).getVal(); return new ObjectBuilder(parser).getVal();
} }
final JSONParser parser; final JSONParser parser;
public ObjectBuilder(JSONParser parser) throws IOException { public ObjectBuilder(JSONParser parser) throws IOException {
this.parser = parser; this.parser = parser;
if (parser.lastEvent()==0) parser.nextEvent(); if (parser.lastEvent()==0) parser.nextEvent();
} }
public Object getVal() throws IOException { public Object getVal() throws IOException {
int ev = parser.lastEvent(); int ev = parser.lastEvent();
switch(ev) { switch(ev) {
case JSONParser.STRING: return getString(); case JSONParser.STRING: return getString();
case JSONParser.LONG: return getLong(); case JSONParser.LONG: return getLong();
case JSONParser.NUMBER: return getNumber(); case JSONParser.NUMBER: return getNumber();
case JSONParser.BIGNUMBER: return getBigNumber(); case JSONParser.BIGNUMBER: return getBigNumber();
case JSONParser.BOOLEAN: return getBoolean(); case JSONParser.BOOLEAN: return getBoolean();
case JSONParser.NULL: return getNull(); case JSONParser.NULL: return getNull();
case JSONParser.OBJECT_START: return getObject(); case JSONParser.OBJECT_START: return getObject();
case JSONParser.OBJECT_END: return null; // OR ERROR? case JSONParser.OBJECT_END: return null; // OR ERROR?
case JSONParser.ARRAY_START: return getArray(); case JSONParser.ARRAY_START: return getArray();
case JSONParser.ARRAY_END: return null; // OR ERROR? case JSONParser.ARRAY_END: return null; // OR ERROR?
case JSONParser.EOF: return null; // OR ERROR? case JSONParser.EOF: return null; // OR ERROR?
default: return null; // OR ERROR? default: return null; // OR ERROR?
} }
} }
public Object getString() throws IOException { public Object getString() throws IOException {
return parser.getString(); return parser.getString();
} }
public Object getLong() throws IOException { public Object getLong() throws IOException {
return Long.valueOf(parser.getLong()); return Long.valueOf(parser.getLong());
} }
public Object getNumber() throws IOException { public Object getNumber() throws IOException {
CharArr num = parser.getNumberChars(); CharArr num = parser.getNumberChars();
String numstr = num.toString(); String numstr = num.toString();
double d = Double.parseDouble(numstr); double d = Double.parseDouble(numstr);
if (!Double.isInfinite(d)) return Double.valueOf(d); if (!Double.isInfinite(d)) return Double.valueOf(d);
// TODO: use more efficient constructor in Java5 // TODO: use more efficient constructor in Java5
return new BigDecimal(numstr); return new BigDecimal(numstr);
} }
public Object getBigNumber() throws IOException { public Object getBigNumber() throws IOException {
CharArr num = parser.getNumberChars(); CharArr num = parser.getNumberChars();
String numstr = num.toString(); String numstr = num.toString();
for(int ch; (ch=num.read())!=-1;) { for(int ch; (ch=num.read())!=-1;) {
if (ch=='.' || ch=='e' || ch=='E') return new BigDecimal(numstr); if (ch=='.' || ch=='e' || ch=='E') return new BigDecimal(numstr);
} }
return new BigInteger(numstr); return new BigInteger(numstr);
} }
public Object getBoolean() throws IOException { public Object getBoolean() throws IOException {
return parser.getBoolean(); return parser.getBoolean();
} }
public Object getNull() throws IOException { public Object getNull() throws IOException {
parser.getNull(); parser.getNull();
return null; return null;
} }
public Object newObject() throws IOException { public Object newObject() throws IOException {
return new LinkedHashMap(); return new LinkedHashMap();
} }
public Object getKey() throws IOException { public Object getKey() throws IOException {
return parser.getString(); return parser.getString();
} }
public void addKeyVal(Object map, Object key, Object val) throws IOException { public void addKeyVal(Object map, Object key, Object val) throws IOException {
Object prev = ((Map)map).put(key,val); Object prev = ((Map)map).put(key,val);
// TODO: test for repeated value? // TODO: test for repeated value?
} }
public Object objectEnd(Object obj) { public Object objectEnd(Object obj) {
return obj; return obj;
} }
public Object getObject() throws IOException { public Object getObject() throws IOException {
Object m = newObject(); Object m = newObject();
for(;;) { for(;;) {
int ev = parser.nextEvent(); int ev = parser.nextEvent();
if (ev==JSONParser.OBJECT_END) return objectEnd(m); if (ev==JSONParser.OBJECT_END) return objectEnd(m);
Object key = getKey(); Object key = getKey();
ev = parser.nextEvent(); ev = parser.nextEvent();
Object val = getVal(); Object val = getVal();
addKeyVal(m, key, val); addKeyVal(m, key, val);
} }
} }
public Object newArray() { public Object newArray() {
return new ArrayList(); return new ArrayList();
} }
public void addArrayVal(Object arr, Object val) throws IOException { public void addArrayVal(Object arr, Object val) throws IOException {
((List)arr).add(val); ((List)arr).add(val);
} }
public Object endArray(Object arr) { public Object endArray(Object arr) {
return arr; return arr;
} }
public Object getArray() throws IOException { public Object getArray() throws IOException {
Object arr = newArray(); Object arr = newArray();
for(;;) { for(;;) {
int ev = parser.nextEvent(); int ev = parser.nextEvent();
if (ev==JSONParser.ARRAY_END) return endArray(arr); if (ev==JSONParser.ARRAY_END) return endArray(arr);
Object val = getVal(); Object val = getVal();
addArrayVal(arr, val); addArrayVal(arr, val);
} }
} }
} }