mirror of https://github.com/apache/lucene.git
move CSVUtil to common from analyzer nori and kuromoji (#12390)
Closes #12389
This commit is contained in:
twosom
GitHub
parent
5b87a31556
commit
30db217048
|
|
@ -14,7 +14,7 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.analysis.ja.dict;
|
||||
package org.apache.lucene.analysis.util;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.regex.Matcher;
|
||||
|
|
@ -69,7 +69,7 @@ public final class CSVUtil {
|
|||
return new String[0];
|
||||
}
|
||||
|
||||
return result.toArray(new String[result.size()]);
|
||||
return result.toArray(new String[0]);
|
||||
}
|
||||
|
||||
private static String unQuoteUnEscape(String original) {
|
||||
|
|
@ -83,7 +83,7 @@ public final class CSVUtil {
|
|||
}
|
||||
|
||||
// Unescape
|
||||
if (result.indexOf(ESCAPED_QUOTE) >= 0) {
|
||||
if (result.contains(ESCAPED_QUOTE)) {
|
||||
result = result.replace(ESCAPED_QUOTE, "\"");
|
||||
}
|
||||
}
|
||||
|
|
@ -14,10 +14,9 @@
|
|||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.analysis.ja;
|
||||
package org.apache.lucene.analysis.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import org.apache.lucene.analysis.ja.dict.CSVUtil;
|
||||
import org.apache.lucene.tests.util.LuceneTestCase;
|
||||
|
||||
/*
|
||||
|
|
@ -28,6 +28,7 @@ import java.util.Comparator;
|
|||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
import org.apache.lucene.util.IntsRefBuilder;
|
||||
import org.apache.lucene.util.fst.FST;
|
||||
import org.apache.lucene.util.fst.FSTCompiler;
|
||||
|
|
|
|||
|
|
@ -20,6 +20,7 @@ import java.io.IOException;
|
|||
import java.io.OutputStream;
|
||||
import java.nio.ByteBuffer;
|
||||
import org.apache.lucene.analysis.morph.DictionaryEntryWriter;
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
import org.apache.lucene.store.DataOutput;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import java.nio.file.Path;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
|
||||
class UnknownDictionaryBuilder {
|
||||
private static final String NGRAM_DICTIONARY_ENTRY = "NGRAM,5,5,-32768,記号,一般,*,*,*,*,*,*,*";
|
||||
|
|
|
|||
|
|
@ -26,6 +26,7 @@ import java.util.List;
|
|||
import java.util.Map;
|
||||
import java.util.TreeMap;
|
||||
import org.apache.lucene.analysis.morph.Dictionary;
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
import org.apache.lucene.util.IntsRefBuilder;
|
||||
import org.apache.lucene.util.fst.FST;
|
||||
import org.apache.lucene.util.fst.FSTCompiler;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ package org.apache.lucene.analysis.ja.dict;
|
|||
import static org.apache.lucene.analysis.ja.dict.UserDictionary.CUSTOM_DICTIONARY_WORD_ID_OFFSET;
|
||||
import static org.apache.lucene.analysis.ja.dict.UserDictionary.INTERNAL_SEPARATOR;
|
||||
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
|
||||
/** Morphological information for user dictionary. */
|
||||
final class UserMorphData implements JaMorphData {
|
||||
public static final int WORD_COST = -100000;
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
*/
|
||||
package org.apache.lucene.analysis.ja.dict;
|
||||
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
import org.apache.lucene.tests.util.LuceneTestCase;
|
||||
import org.junit.Test;
|
||||
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
package org.apache.lucene.analysis.ko.dict;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/** Utility class for parsing CSV text */
|
||||
public final class CSVUtil {
|
||||
private static final char QUOTE = '"';
|
||||
|
||||
private static final char COMMA = ',';
|
||||
|
||||
private static final Pattern QUOTE_REPLACE_PATTERN = Pattern.compile("^\"([^\"]+)\"$");
|
||||
|
||||
private static final String ESCAPED_QUOTE = "\"\"";
|
||||
|
||||
private CSVUtil() {} // no instance!!!
|
||||
|
||||
/**
|
||||
* Parse CSV line
|
||||
*
|
||||
* @param line line containing csv-encoded data
|
||||
* @return Array of values
|
||||
*/
|
||||
public static String[] parse(String line) {
|
||||
boolean insideQuote = false;
|
||||
ArrayList<String> result = new ArrayList<>();
|
||||
int quoteCount = 0;
|
||||
StringBuilder sb = new StringBuilder();
|
||||
for (int i = 0; i < line.length(); i++) {
|
||||
char c = line.charAt(i);
|
||||
|
||||
if (c == QUOTE) {
|
||||
insideQuote = !insideQuote;
|
||||
quoteCount++;
|
||||
}
|
||||
|
||||
if (c == COMMA && !insideQuote) {
|
||||
String value = sb.toString();
|
||||
value = unQuoteUnEscape(value);
|
||||
result.add(value);
|
||||
sb.setLength(0);
|
||||
continue;
|
||||
}
|
||||
|
||||
sb.append(c);
|
||||
}
|
||||
|
||||
result.add(sb.toString());
|
||||
|
||||
// Validate
|
||||
if (quoteCount % 2 != 0) {
|
||||
return new String[0];
|
||||
}
|
||||
|
||||
return result.toArray(new String[0]);
|
||||
}
|
||||
|
||||
private static String unQuoteUnEscape(String original) {
|
||||
String result = original;
|
||||
|
||||
// Unquote
|
||||
if (result.indexOf('\"') >= 0) {
|
||||
Matcher m = QUOTE_REPLACE_PATTERN.matcher(original);
|
||||
if (m.matches()) {
|
||||
result = m.group(1);
|
||||
}
|
||||
|
||||
// Unescape
|
||||
if (result.contains(ESCAPED_QUOTE)) {
|
||||
result = result.replace(ESCAPED_QUOTE, "\"");
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
|
@ -28,6 +28,7 @@ import java.util.Comparator;
|
|||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
import org.apache.lucene.util.IntsRefBuilder;
|
||||
import org.apache.lucene.util.fst.FST;
|
||||
import org.apache.lucene.util.fst.FSTCompiler;
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import java.util.Arrays;
|
|||
import java.util.List;
|
||||
import org.apache.lucene.analysis.ko.POS;
|
||||
import org.apache.lucene.analysis.morph.DictionaryEntryWriter;
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
import org.apache.lucene.store.DataOutput;
|
||||
import org.apache.lucene.util.ArrayUtil;
|
||||
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ import java.nio.file.Path;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Comparator;
|
||||
import java.util.List;
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
|
||||
class UnknownDictionaryBuilder {
|
||||
private static final String NGRAM_DICTIONARY_ENTRY = "NGRAM,1801,3559,3677,SY,*,*,*,*,*,*,*";
|
||||
|
|
|
|||
|
|
@ -16,6 +16,7 @@
|
|||
*/
|
||||
package org.apache.lucene.analysis.ko.dict;
|
||||
|
||||
import org.apache.lucene.analysis.util.CSVUtil;
|
||||
import org.apache.lucene.tests.util.LuceneTestCase;
|
||||
import org.junit.Test;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue