mirror of https://github.com/apache/lucene.git
Russian and German analyzers have been moved to sandbox: delete the test cases here, too
git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@150435 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Daniel Naber
parent
caaad35b57
commit
7ec48d172c
|
|
@ -1,78 +0,0 @@
|
|||
package org.apache.lucene.analysis.de;
|
||||
|
||||
/**
|
||||
* Copyright 2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.StringReader;
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import org.apache.lucene.analysis.Token;
|
||||
import org.apache.lucene.analysis.standard.StandardTokenizer;
|
||||
|
||||
/**
|
||||
* Test the German stemmer. The stemming algorithm is known to work less
|
||||
* than perfect, as it doesn't use any word lists with exceptions. We
|
||||
* also check some of the cases where the algorithm is wrong.
|
||||
*
|
||||
* @author Daniel Naber
|
||||
*/
|
||||
public class TestGermanStemFilter extends TestCase {
|
||||
|
||||
public void testStemming() {
|
||||
try {
|
||||
// read test cases from external file:
|
||||
File dataDir = new File(System.getProperty("dataDir", "./bin"));
|
||||
File testFile = new File(dataDir, "org/apache/lucene/analysis/de/data.txt");
|
||||
FileInputStream fis = new FileInputStream(testFile);
|
||||
InputStreamReader isr = new InputStreamReader(fis, "iso-8859-1");
|
||||
BufferedReader breader = new BufferedReader(isr);
|
||||
while(true) {
|
||||
String line = breader.readLine();
|
||||
if (line == null)
|
||||
break;
|
||||
line = line.trim();
|
||||
if (line.startsWith("#") || line.equals(""))
|
||||
continue; // ignore comments and empty lines
|
||||
String[] parts = line.split(";");
|
||||
//System.out.println(parts[0] + " -- " + parts[1]);
|
||||
check(parts[0], parts[1]);
|
||||
}
|
||||
breader.close();
|
||||
isr.close();
|
||||
fis.close();
|
||||
} catch (IOException e) {
|
||||
e.printStackTrace();
|
||||
fail();
|
||||
}
|
||||
}
|
||||
|
||||
private void check(final String input, final String expected) throws IOException {
|
||||
StandardTokenizer tokenStream = new StandardTokenizer(new StringReader(input));
|
||||
GermanStemFilter filter = new GermanStemFilter(tokenStream);
|
||||
Token t = filter.next();
|
||||
if (t == null)
|
||||
fail();
|
||||
assertEquals(expected, t.termText());
|
||||
filter.close();
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1,48 +0,0 @@
|
|||
# German special characters are replaced:
|
||||
häufig;haufig
|
||||
|
||||
# here the stemmer works okay, it maps related words to the same stem:
|
||||
abschließen;abschliess
|
||||
abschließender;abschliess
|
||||
abschließendes;abschliess
|
||||
abschließenden;abschliess
|
||||
|
||||
Tisch;tisch
|
||||
Tische;tisch
|
||||
Tischen;tisch
|
||||
|
||||
Haus;hau
|
||||
Hauses;hau
|
||||
Häuser;hau
|
||||
Häusern;hau
|
||||
# here's a case where overstemming occurs, i.e. a word is
|
||||
# mapped to the same stem as unrelated words:
|
||||
hauen;hau
|
||||
|
||||
# here's a case where understemming occurs, i.e. two related words
|
||||
# are not mapped to the same stem. This is the case with basically
|
||||
# all irregular forms:
|
||||
Drama;drama
|
||||
Dramen;dram
|
||||
|
||||
# replace "ß" with 'ss':
|
||||
Ausmaß;ausmass
|
||||
|
||||
# fake words to test if suffixes are cut off:
|
||||
xxxxxe;xxxxx
|
||||
xxxxxs;xxxxx
|
||||
xxxxxn;xxxxx
|
||||
xxxxxt;xxxxx
|
||||
xxxxxem;xxxxx
|
||||
xxxxxer;xxxxx
|
||||
xxxxxnd;xxxxx
|
||||
# the suffixes are also removed when combined:
|
||||
xxxxxetende;xxxxx
|
||||
|
||||
# words that are shorter than four charcters are not changed:
|
||||
xxe;xxe
|
||||
# -em and -er are not removed from words shorter than five characters:
|
||||
xxem;xxem
|
||||
xxer;xxer
|
||||
# -nd is not removed from words shorter than six characters:
|
||||
xxxnd;xxxnd
|
||||
|
|
@ -1,170 +0,0 @@
|
|||
package org.apache.lucene.analysis.ru;
|
||||
|
||||
/**
|
||||
* Copyright 2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import java.io.*;
|
||||
|
||||
import org.apache.lucene.analysis.TokenStream;
|
||||
import org.apache.lucene.analysis.Token;
|
||||
|
||||
/**
|
||||
* Test case for RussianAnalyzer.
|
||||
*
|
||||
* @author Boris Okner
|
||||
* @version $Id$
|
||||
*/
|
||||
|
||||
public class TestRussianAnalyzer extends TestCase
|
||||
{
|
||||
private InputStreamReader inWords;
|
||||
|
||||
private InputStreamReader sampleUnicode;
|
||||
|
||||
private Reader inWordsKOI8;
|
||||
|
||||
private Reader sampleKOI8;
|
||||
|
||||
private Reader inWords1251;
|
||||
|
||||
private Reader sample1251;
|
||||
|
||||
private File dataDir;
|
||||
|
||||
protected void setUp() throws Exception
|
||||
{
|
||||
dataDir = new File(System.getProperty("dataDir", "./bin"));
|
||||
}
|
||||
|
||||
public void testUnicode() throws IOException
|
||||
{
|
||||
RussianAnalyzer ra = new RussianAnalyzer(RussianCharsets.UnicodeRussian);
|
||||
inWords =
|
||||
new InputStreamReader(
|
||||
new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/testUnicode.txt")),
|
||||
"Unicode");
|
||||
|
||||
sampleUnicode =
|
||||
new InputStreamReader(
|
||||
new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/resUnicode.htm")),
|
||||
"Unicode");
|
||||
|
||||
TokenStream in = ra.tokenStream("all", inWords);
|
||||
|
||||
RussianLetterTokenizer sample =
|
||||
new RussianLetterTokenizer(
|
||||
sampleUnicode,
|
||||
RussianCharsets.UnicodeRussian);
|
||||
|
||||
for (;;)
|
||||
{
|
||||
Token token = in.next();
|
||||
|
||||
if (token == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
Token sampleToken = sample.next();
|
||||
assertEquals(
|
||||
"Unicode",
|
||||
token.termText(),
|
||||
sampleToken == null
|
||||
? null
|
||||
: sampleToken.termText());
|
||||
}
|
||||
|
||||
inWords.close();
|
||||
sampleUnicode.close();
|
||||
}
|
||||
|
||||
public void testKOI8() throws IOException
|
||||
{
|
||||
//System.out.println(new java.util.Date());
|
||||
RussianAnalyzer ra = new RussianAnalyzer(RussianCharsets.KOI8);
|
||||
// KOI8
|
||||
inWordsKOI8 = new InputStreamReader(new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/testKOI8.txt")), "iso-8859-1");
|
||||
|
||||
sampleKOI8 = new InputStreamReader(new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/resKOI8.htm")), "iso-8859-1");
|
||||
|
||||
TokenStream in = ra.tokenStream("all", inWordsKOI8);
|
||||
RussianLetterTokenizer sample =
|
||||
new RussianLetterTokenizer(
|
||||
sampleKOI8,
|
||||
RussianCharsets.KOI8);
|
||||
|
||||
for (;;)
|
||||
{
|
||||
Token token = in.next();
|
||||
|
||||
if (token == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
Token sampleToken = sample.next();
|
||||
assertEquals(
|
||||
"KOI8",
|
||||
token.termText(),
|
||||
sampleToken == null
|
||||
? null
|
||||
: sampleToken.termText());
|
||||
|
||||
}
|
||||
|
||||
inWordsKOI8.close();
|
||||
sampleKOI8.close();
|
||||
}
|
||||
|
||||
public void test1251() throws IOException
|
||||
{
|
||||
// 1251
|
||||
inWords1251 = new InputStreamReader(new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/test1251.txt")), "iso-8859-1");
|
||||
|
||||
sample1251 = new InputStreamReader(new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/res1251.htm")), "iso-8859-1");
|
||||
|
||||
RussianAnalyzer ra = new RussianAnalyzer(RussianCharsets.CP1251);
|
||||
TokenStream in = ra.tokenStream("", inWords1251);
|
||||
RussianLetterTokenizer sample =
|
||||
new RussianLetterTokenizer(
|
||||
sample1251,
|
||||
RussianCharsets.CP1251);
|
||||
|
||||
for (;;)
|
||||
{
|
||||
Token token = in.next();
|
||||
|
||||
if (token == null)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
Token sampleToken = sample.next();
|
||||
assertEquals(
|
||||
"1251",
|
||||
token.termText(),
|
||||
sampleToken == null
|
||||
? null
|
||||
: sampleToken.termText());
|
||||
|
||||
}
|
||||
|
||||
inWords1251.close();
|
||||
sample1251.close();
|
||||
}
|
||||
}
|
||||
|
|
@ -1,94 +0,0 @@
|
|||
package org.apache.lucene.analysis.ru;
|
||||
|
||||
/**
|
||||
* Copyright 2004 The Apache Software Foundation
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import junit.framework.TestCase;
|
||||
|
||||
import java.io.BufferedReader;
|
||||
import java.io.File;
|
||||
import java.io.InputStreamReader;
|
||||
import java.io.FileInputStream;
|
||||
import java.util.ArrayList;
|
||||
|
||||
public class TestRussianStem extends TestCase
|
||||
{
|
||||
private ArrayList words = new ArrayList();
|
||||
private ArrayList stems = new ArrayList();
|
||||
|
||||
public TestRussianStem(String name)
|
||||
{
|
||||
super(name);
|
||||
}
|
||||
|
||||
/**
|
||||
* @see TestCase#setUp()
|
||||
*/
|
||||
protected void setUp() throws Exception
|
||||
{
|
||||
super.setUp();
|
||||
//System.out.println(new java.util.Date());
|
||||
String str;
|
||||
|
||||
File dataDir = new File(System.getProperty("dataDir"));
|
||||
|
||||
// open and read words into an array list
|
||||
BufferedReader inWords =
|
||||
new BufferedReader(
|
||||
new InputStreamReader(
|
||||
new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/wordsUnicode.txt")),
|
||||
"Unicode"));
|
||||
while ((str = inWords.readLine()) != null)
|
||||
{
|
||||
words.add(str);
|
||||
}
|
||||
inWords.close();
|
||||
|
||||
// open and read stems into an array list
|
||||
BufferedReader inStems =
|
||||
new BufferedReader(
|
||||
new InputStreamReader(
|
||||
new FileInputStream(new File(dataDir, "/org/apache/lucene/analysis/ru/stemsUnicode.txt")),
|
||||
"Unicode"));
|
||||
while ((str = inStems.readLine()) != null)
|
||||
{
|
||||
stems.add(str);
|
||||
}
|
||||
inStems.close();
|
||||
}
|
||||
|
||||
/**
|
||||
* @see TestCase#tearDown()
|
||||
*/
|
||||
protected void tearDown() throws Exception
|
||||
{
|
||||
super.tearDown();
|
||||
}
|
||||
|
||||
public void testStem()
|
||||
{
|
||||
for (int i = 0; i < words.size(); i++)
|
||||
{
|
||||
//if ( (i % 100) == 0 ) System.err.println(i);
|
||||
String realStem =
|
||||
RussianStemmer.stem(
|
||||
(String) words.get(i),
|
||||
RussianCharsets.UnicodeRussian);
|
||||
assertEquals("unicode", stems.get(i), realStem);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -1 +0,0 @@
|
|||
[вмест][сил][электромагнитн][энерг][имел][представлен][скаж][жрец][древн][египт][знан][хран][тайн][узк][круг][посвящен][всяк][времен][виток][прин][соб][нов][технолог][сам][дел][раскрыва][потаен][знан][прежн][век][говор][нов][информац][станов][доступн][широк][круг][пользовател][тех][случа][сознан][обществ][готов][восприня][воспользова]
|
||||
|
|
@ -1 +0,0 @@
|
|||
[淄庞註[由蘛[芴潘砸贤燎紊晕][芪乓荾[赏盘][幸拍釉磷膛蝅[铀林][忠琶][囊抛蝅[徘尚註[谖廖][纫廖][粤饰][遮薦[艘涨][邢幼演盼][子阉][滓磐盼][咨韵薦[幸晌][酉耛[蜗譣[耘任咸锨][恿蚞[呐蘛[伊铀屹琢][邢粤盼][谖廖][幸胖蝅[着薦[窍紫襗[蜗譣[晌葡彝撩][釉廖献][南釉招蝅[凵蚁薦[艘涨][邢特谙琢耘蘛[耘萞[犹辙羃[酉谖廖][下菖釉譣[窍韵譣[紫有疑窝][紫有咸刳献羃
|
||||
Binary file not shown.
Binary file not shown.
|
|
@ -1,2 +0,0 @@
|
|||
Вместе с тем о силе электромагнитной энергии имели представление еще, скажем, жрецы Древнего Египта. Но знание это хранилось в тайне, в
|
||||
узком кругу посвященных. Всякий временной виток, принося с собой новые технологии, на самом деле раскрывает потаенное знание прежних веков. Мы уже говорили, что новая информация становится доступной широкому кругу пользователей только в тех случаях, когда сознание общества готово ее воспринять и воспользоваться ею.
|
||||
|
|
@ -1,2 +0,0 @@
|
|||
Вместе с тем о силе электромагнитной энергии имели представление еще, скажем, жрецы Древнего Египта. Но знание это хранилось в тайне, в
|
||||
узком кругу посвященных. Всякий временной виток, принося с собой новые технологии, на самом деле раскрывает потаенное знание прежних веков. Мы уже говорили, что новая информация становится доступной широкому кругу пользователей только в тех случаях, когда сознание общества готово ее воспринять и воспользоваться ею.
|
||||
Binary file not shown.
Binary file not shown.
Loading…
Reference in New Issue