Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-10335: Deprecate helper methods for resource loading in IOUtils and StopwordAnalyzerBase that are not compatible with module system; add utility method IOUtils#requireResourceNonNull(T) and add ModuleResourceLoader as complement to ClasspathResourceLoader 

Co-authored-by: Tomoko Uchida <tomoko.uchida.1111@gmail.com>
This commit is contained in:
Uwe Schindler 2022-01-03 10:38:19 +01:00
parent df4562d149
commit 3ec3456d21
65 changed files with 588 additions and 220 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
lucene/CHANGES.txt
View File

@ -18,6 +18,11 @@ API Changes
org.apache.lucene.* to org.apache.lucene.tests.* to avoid package name conflicts with the
core module. (Dawid Weiss)
* LUCENE-10335: Deprecate helper methods for resource loading in IOUtils and StopwordAnalyzerBase
that are not compatible with module system (Class#getResourceAsStream() and Class#getResource()
are caller sensitive in Java 11). Instead add utility method IOUtils#requireResourceNonNull(T)
to test existence of resource based on null return value. (Uwe Schindler, Dawid Weiss)
New Features
---------------------
@ -56,6 +61,9 @@ New Features
* LUCENE-10243: StandardTokenizer, UAX29URLEmailTokenizer, and HTMLStripCharFilter have
been upgraded to Unicode 12.1 (Robert Muir)
* LUCENE-10335: Add ModuleResourceLoader as complement to ClasspathResourceLoader.
(Uwe Schindler)
Improvements
---------------------

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
View File

@ -26,9 +26,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* {@link Analyzer} for Arabic.
@ -74,7 +76,12 @@ public final class ArabicAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = loadStopwordSet(false, ArabicAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
ArabicAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/bg/BulgarianAnalyzer.java
View File

@ -26,8 +26,10 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* {@link Analyzer} for Bulgarian.
@ -66,7 +68,11 @@ public final class BulgarianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, BulgarianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
BulgarianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/bn/BengaliAnalyzer.java
View File

@ -25,10 +25,12 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.in.IndicNormalizationFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* Analyzer for Bengali.
@ -67,7 +69,11 @@ public final class BengaliAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, BengaliAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
BengaliAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
throw new UncheckedIOException("Unable to load default stopword set", ex);
}

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/br/BrazilianAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.br;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -64,8 +63,9 @@ public final class BrazilianAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
BrazilianAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
IOUtils.requireResourceNonNull(
BrazilianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/ca/CatalanAnalyzer.java
View File

@ -27,10 +27,12 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.CatalanStemmer;
/**
@ -67,7 +69,11 @@ public final class CatalanAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, CatalanAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
CatalanAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKAnalyzer.java
View File

@ -25,7 +25,9 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* An {@link Analyzer} that tokenizes text with {@link StandardTokenizer}, normalizes content with
@ -58,7 +60,12 @@ public final class CJKAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = loadStopwordSet(false, CJKAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
CJKAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/ckb/SoraniAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.ckb;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -64,8 +63,9 @@ public final class SoraniAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
SoraniAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SoraniAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/cz/CzechAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.cz;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -60,8 +59,9 @@ public final class CzechAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_SET =
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
CzechAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
IOUtils.requireResourceNonNull(
CzechAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/da/DanishAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.da;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -65,8 +64,9 @@ public final class DanishAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/de/GermanAnalyzer.java
View File

@ -20,7 +20,6 @@ package org.apache.lucene.analysis.de;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -75,8 +74,9 @@ public final class GermanAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/el/GreekAnalyzer.java
View File

@ -25,8 +25,10 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* {@link Analyzer} for the Greek language.
@ -57,7 +59,12 @@ public final class GreekAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_SET = loadStopwordSet(false, GreekAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
DEFAULT_SET =
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
GreekAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/es/SpanishAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.es;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -64,8 +63,9 @@ public final class SpanishAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/et/EstonianAnalyzer.java
View File

@ -26,9 +26,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.EstonianStemmer;
/** {@link Analyzer} for Estonian. */
@ -57,7 +59,11 @@ public final class EstonianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, EstonianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
EstonianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/eu/BasqueAnalyzer.java
View File

@ -26,9 +26,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.BasqueStemmer;
/**
@ -60,7 +62,12 @@ public final class BasqueAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = loadStopwordSet(false, BasqueAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
BasqueAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
View File

@ -26,9 +26,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* {@link Analyzer} for Persian.
@ -71,7 +73,11 @@ public final class PersianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, PersianAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
PersianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/fi/FinnishAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.fi;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -65,8 +64,9 @@ public final class FinnishAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.fr;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
@ -81,8 +80,9 @@ public final class FrenchAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java
View File

@ -26,10 +26,12 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.IrishStemmer;
/**
@ -72,7 +74,12 @@ public final class IrishAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET = loadStopwordSet(false, IrishAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
IrishAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/gl/GalicianAnalyzer.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.analysis.gl;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -62,8 +61,9 @@ public final class GalicianAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
GalicianAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
GalicianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/hi/HindiAnalyzer.java
View File

@ -25,10 +25,12 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.in.IndicNormalizationFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* Analyzer for Hindi.
@ -67,7 +69,11 @@ public final class HindiAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, HindiAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
HindiAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/hu/HungarianAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.hu;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -65,8 +64,9 @@ public final class HungarianAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/hy/ArmenianAnalyzer.java
View File

@ -26,9 +26,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.ArmenianStemmer;
/**
@ -61,7 +63,11 @@ public final class ArmenianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, ArmenianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
ArmenianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/id/IndonesianAnalyzer.java
View File

@ -25,8 +25,10 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* Analyzer for Indonesian (Bahasa)
@ -56,7 +58,11 @@ public final class IndonesianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, IndonesianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
IndonesianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.it;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
@ -74,8 +73,9 @@ public final class ItalianAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/lt/LithuanianAnalyzer.java
View File

@ -26,9 +26,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.LithuanianStemmer;
/**
@ -61,7 +63,11 @@ public final class LithuanianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, LithuanianAnalyzer.class, DEFAULT_STOPWORD_FILE, "#");
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
LithuanianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/lv/LatvianAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.lv;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -63,8 +62,9 @@ public final class LatvianAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
LatvianAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
LatvianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/ne/NepaliAnalyzer.java
View File

@ -25,11 +25,13 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.in.IndicNormalizationFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.NepaliStemmer;
/**
@ -69,7 +71,11 @@ public final class NepaliAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, NepaliAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
NepaliAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/nl/DutchAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.nl;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArrayMap;
import org.apache.lucene.analysis.CharArraySet;
@ -69,8 +68,9 @@ public final class DutchAnalyzer extends Analyzer {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/no/NorwegianAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.no;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -65,8 +64,9 @@ public final class NorwegianAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/PortugueseAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.pt;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -64,8 +63,9 @@ public final class PortugueseAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/ro/RomanianAnalyzer.java
View File

@ -26,9 +26,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.RomanianStemmer;
/**
@ -63,8 +65,11 @@ public final class RomanianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(
false, RomanianAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
RomanianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/ru/RussianAnalyzer.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.analysis.ru;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -52,8 +51,9 @@ public final class RussianAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

7
lucene/analysis/common/src/java/org/apache/lucene/analysis/sr/SerbianAnalyzer.java
View File

@ -23,6 +23,7 @@ import org.apache.lucene.analysis.*;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.SerbianStemmer;
/**
@ -58,7 +59,11 @@ public class SerbianAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, SerbianAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
SerbianAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

6
lucene/analysis/common/src/java/org/apache/lucene/analysis/sv/SwedishAnalyzer.java
View File

@ -19,7 +19,6 @@ package org.apache.lucene.analysis.sv;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -65,8 +64,9 @@ public final class SwedishAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(
SnowballFilter.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8));
IOUtils.requireResourceNonNull(
SnowballFilter.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE));
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/ta/TamilAnalyzer.java
View File

@ -25,11 +25,13 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.in.IndicNormalizationFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.TamilStemmer;
/**
@ -68,7 +70,11 @@ public final class TamilAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, TamilAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
TamilAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/te/TeluguAnalyzer.java
View File

@ -24,10 +24,12 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.analysis.in.IndicNormalizationFilter;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
/**
* Analyzer for Telugu.
@ -61,7 +63,11 @@ public final class TeluguAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, TeluguAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
TeluguAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
throw new UncheckedIOException("Unable to load default stopword set", ex);

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiAnalyzer.java
View File

@ -26,7 +26,9 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.core.DecimalDigitFilter;
import org.apache.lucene.util.IOUtils;
/**
* {@link Analyzer} for Thai language. It uses {@link java.text.BreakIterator} to break words.
@ -59,7 +61,11 @@ public final class ThaiAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, ThaiAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
ThaiAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

8
lucene/analysis/common/src/java/org/apache/lucene/analysis/tr/TurkishAnalyzer.java
View File

@ -25,9 +25,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.miscellaneous.SetKeywordMarkerFilter;
import org.apache.lucene.analysis.snowball.SnowballFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.util.IOUtils;
import org.tartarus.snowball.ext.TurkishStemmer;
/**
@ -62,7 +64,11 @@ public final class TurkishAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(false, TurkishAnalyzer.class, DEFAULT_STOPWORD_FILE, STOPWORDS_COMMENT);
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
TurkishAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORDS_COMMENT);
} catch (IOException ex) {
// default set should always be present as it is part of the
// distribution (JAR)

34
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/FilesystemResourceLoader.java
View File

@ -22,7 +22,9 @@ import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
import java.util.Objects;
import org.apache.lucene.util.ClasspathResourceLoader;
import org.apache.lucene.util.ModuleResourceLoader;
import org.apache.lucene.util.ResourceLoader;
/**
@ -41,24 +43,36 @@ public final class FilesystemResourceLoader implements ResourceLoader {
private final ResourceLoader delegate;
/**
* Creates a resource loader that resolves resources against the given base directory (may be
* {@code null} to refer to CWD). Files not found in file system and class lookups are delegated
* to context classloader.
* Creates a resource loader that resolves resources against the given base directory. Files not
* found in file system and class lookups are delegated to {@link ClassLoader}.
*
* <p>To use this constructor with the Java Module System, you must open all modules that contain
* resources to the {@code org.apache.lucene.core} module, otherwise resources can't be looked up.
* It is recommended to use {@link #FilesystemResourceLoader(Path, Module)} for such use cases as
* this would limit to certain modules.
*/
public FilesystemResourceLoader(Path baseDirectory, ClassLoader delegate) {
this(baseDirectory, new ClasspathResourceLoader(delegate));
}
/**
* Creates a resource loader that resolves resources against the given base directory (may be
* {@code null} to refer to CWD). Files not found in file system and class lookups are delegated
* to the given delegate {@link ResourceLoader}.
* Creates a resource loader that resolves resources against the given base directory. Files not
* found in file system and class lookups are delegated to {@link ModuleResourceLoader}.
*
* <p>To use this constructor, you must open the module to the {@code org.apache.lucene.core}
* module, otherwise resources can't be looked up.
*/
public FilesystemResourceLoader(Path baseDirectory, Module delegate) {
this(baseDirectory, new ModuleResourceLoader(delegate));
}
/**
* Creates a resource loader that resolves resources against the given base directory. Files not
* found in file system and class lookups are delegated to the given delegate {@link
* ResourceLoader}.
*/
public FilesystemResourceLoader(Path baseDirectory, ResourceLoader delegate) {
if (baseDirectory == null) {
throw new NullPointerException();
}
if (!Files.isDirectory(baseDirectory))
if (!Files.isDirectory(Objects.requireNonNull(baseDirectory)))
throw new IllegalArgumentException(baseDirectory + " is not a directory");
if (delegate == null)
throw new IllegalArgumentException("delegate ResourceLoader may not be null");

10
lucene/analysis/common/src/test/org/apache/lucene/analysis/util/StringMockResourceLoader.java
View File

@ -39,16 +39,6 @@ public class StringMockResourceLoader implements ResourceLoader {
}
}
@Override
public <T> T newInstance(String cname, Class<T> expectedType) {
Class<? extends T> clazz = findClass(cname, expectedType);
try {
return clazz.getConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("Cannot create instance: " + cname, e);
}
}
@Override
public InputStream openResource(String resource) throws IOException {
return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8));

17
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/JapaneseAnalyzer.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.analysis.ja;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.analysis.CharArraySet;
@ -27,9 +28,11 @@ import org.apache.lucene.analysis.StopFilter;
import org.apache.lucene.analysis.StopwordAnalyzerBase;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.WordlistLoader;
import org.apache.lucene.analysis.cjk.CJKWidthCharFilter;
import org.apache.lucene.analysis.ja.JapaneseTokenizer.Mode;
import org.apache.lucene.analysis.ja.dict.UserDictionary;
import org.apache.lucene.util.IOUtils;
/**
* Analyzer for Japanese that uses morphological analysis.
@ -77,9 +80,19 @@ public class JapaneseAnalyzer extends StopwordAnalyzerBase {
static {
try {
DEFAULT_STOP_SET =
loadStopwordSet(true, JapaneseAnalyzer.class, "stopwords.txt", "#"); // ignore case
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
IOUtils.requireResourceNonNull(
JapaneseAnalyzer.class.getResourceAsStream("stopwords.txt"),
"stopwords.txt"),
StandardCharsets.UTF_8),
"#",
new CharArraySet(16, true)); // ignore case
final CharArraySet tagset =
loadStopwordSet(false, JapaneseAnalyzer.class, "stoptags.txt", "#");
WordlistLoader.getWordSet(
IOUtils.requireResourceNonNull(
JapaneseAnalyzer.class.getResourceAsStream("stoptags.txt"), "stoptags.txt"),
"#");
DEFAULT_STOP_TAGS = new HashSet<>();
for (Object element : tagset) {
char[] chars = (char[]) element;

11
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/BinaryDictionary.java
View File

@ -75,8 +75,11 @@ public abstract class BinaryDictionary implements Dictionary {
throw new IllegalArgumentException(
"resourcePath must be supplied with FILE resource scheme");
}
this.resourcePath = getClass().getName().replace('.', '/');
this.resourcePath = getClass().getSimpleName();
} else {
if (resourceScheme == ResourceScheme.CLASSPATH && !resourcePath.startsWith("/")) {
resourcePath = "/".concat(resourcePath);
}
this.resourcePath = resourcePath;
}
InputStream mapIS = null, dictIS = null, posIS = null;
@ -204,11 +207,7 @@ public abstract class BinaryDictionary implements Dictionary {
}
private static InputStream getClassResource(String path) throws IOException {
final InputStream is = BinaryDictionary.class.getClassLoader().getResourceAsStream(path);
if (is == null) {
throw new FileNotFoundException("Not in classpath: " + path);
}
return is;
return IOUtils.requireResourceNonNull(BinaryDictionary.class.getResourceAsStream(path), path);
}
public void lookupWordIds(int sourceId, IntsRef ref) {

16
lucene/analysis/kuromoji/src/java/org/apache/lucene/analysis/ja/dict/ConnectionCosts.java
View File

@ -23,7 +23,6 @@ import java.nio.ByteBuffer;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOUtils;
/** n-gram connection cost data */
public final class ConnectionCosts {
@ -40,11 +39,9 @@ public final class ConnectionCosts {
* @param path - where to load resources from, without the ".dat" suffix
*/
public ConnectionCosts(BinaryDictionary.ResourceScheme scheme, String path) throws IOException {
InputStream is = null;
boolean success = false;
try {
is = BinaryDictionary.getResource(scheme, path.replace('.', '/') + FILENAME_SUFFIX);
is = new BufferedInputStream(is);
try (InputStream is =
new BufferedInputStream(
BinaryDictionary.getResource(scheme, "/" + path.replace('.', '/') + FILENAME_SUFFIX))) {
final DataInput in = new InputStreamDataInput(is);
CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
forwardSize = in.readVInt();
@ -61,13 +58,6 @@ public final class ConnectionCosts {
}
}
buffer = tmpBuffer.asReadOnlyBuffer();
success = true;
} finally {
if (success) {
IOUtils.close(is);
} else {
IOUtils.closeWhileHandlingException(is);
}
}
}

10
lucene/analysis/kuromoji/src/test/org/apache/lucene/analysis/ja/StringMockResourceLoader.java
View File

@ -39,16 +39,6 @@ class StringMockResourceLoader implements ResourceLoader {
}
}
@Override
public <T> T newInstance(String cname, Class<T> expectedType) {
Class<? extends T> clazz = findClass(cname, expectedType);
try {
return clazz.getConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("Cannot create instance: " + cname, e);
}
}
@Override
public InputStream openResource(String resource) throws IOException {
return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8));

15
lucene/analysis/morfologik/src/java/org/apache/lucene/analysis/uk/UkrainianMorfologikAnalyzer.java
View File

@ -19,8 +19,6 @@ package org.apache.lucene.analysis.uk;
import java.io.IOException;
import java.io.Reader;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Objects;
import morfologik.stemming.Dictionary;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
@ -77,9 +75,7 @@ public final class UkrainianMorfologikAnalyzer extends StopwordAnalyzerBase {
if (is == null) {
throw new IOException("Could not locate the required stopwords resource.");
}
wordList =
WordlistLoader.getSnowballWordSet(
IOUtils.getDecodingReader(is, StandardCharsets.UTF_8));
wordList = WordlistLoader.getSnowballWordSet(is);
}
// First, try to look up the resource module by name.
@ -100,13 +96,12 @@ public final class UkrainianMorfologikAnalyzer extends StopwordAnalyzerBase {
dictionary = Dictionary.read(fsaStream, metaStream);
}
} else {
var name = "ua/net/nlp/ukrainian.dict";
dictionary =
Dictionary.read(
Objects.requireNonNull(
UkrainianMorfologikAnalyzer.class
.getClassLoader()
.getResource("ua/net/nlp/ukrainian.dict"),
"Could not locate the required Ukrainian dictionary resource."));
IOUtils.requireResourceNonNull(
UkrainianMorfologikAnalyzer.class.getClassLoader().getResource(name),
name));
}
defaultResources = new DefaultResources(wordList, dictionary);
} catch (IOException e) {

5
lucene/analysis/morfologik/src/test/org/apache/lucene/analysis/morfologik/TestMorfologikFilterFactory.java
View File

@ -39,11 +39,6 @@ public class TestMorfologikFilterFactory extends BaseTokenStreamTestCase {
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
throw new UnsupportedOperationException();
}
@Override
public <T> T newInstance(String cname, Class<T> expectedType) {
throw new UnsupportedOperationException();
}
}
public void testDefaultDictionary() throws Exception {

11
lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/BinaryDictionary.java
View File

@ -74,8 +74,11 @@ public abstract class BinaryDictionary implements Dictionary {
throw new IllegalArgumentException(
"resourcePath must be supplied with FILE resource scheme");
}
this.resourcePath = getClass().getName().replace('.', '/');
this.resourcePath = getClass().getSimpleName();
} else {
if (resourceScheme == ResourceScheme.CLASSPATH && !resourcePath.startsWith("/")) {
resourcePath = "/".concat(resourcePath);
}
this.resourcePath = resourcePath;
}
InputStream mapIS = null, dictIS = null, posIS = null;
@ -178,11 +181,7 @@ public abstract class BinaryDictionary implements Dictionary {
}
private static InputStream getClassResource(String path) throws IOException {
final InputStream is = BinaryDictionary.class.getClassLoader().getResourceAsStream(path);
if (is == null) {
throw new FileNotFoundException("Not in classpath: " + path);
}
return is;
return IOUtils.requireResourceNonNull(BinaryDictionary.class.getResourceAsStream(path), path);
}
public void lookupWordIds(int sourceId, IntsRef ref) {

17
lucene/analysis/nori/src/java/org/apache/lucene/analysis/ko/dict/ConnectionCosts.java
View File

@ -23,7 +23,6 @@ import java.nio.ByteBuffer;
import org.apache.lucene.codecs.CodecUtil;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.util.IOUtils;
/** n-gram connection cost data */
public final class ConnectionCosts {
@ -41,11 +40,10 @@ public final class ConnectionCosts {
*/
public ConnectionCosts(BinaryDictionary.ResourceScheme scheme, String resourcePath)
throws IOException {
InputStream is = null;
boolean success = false;
try {
is = BinaryDictionary.getResource(scheme, resourcePath.replace('.', '/') + FILENAME_SUFFIX);
is = new BufferedInputStream(is);
try (InputStream is =
new BufferedInputStream(
BinaryDictionary.getResource(
scheme, "/" + resourcePath.replace('.', '/') + FILENAME_SUFFIX))) {
final DataInput in = new InputStreamDataInput(is);
CodecUtil.checkHeader(in, HEADER, VERSION, VERSION);
this.forwardSize = in.readVInt();
@ -62,13 +60,6 @@ public final class ConnectionCosts {
}
}
buffer = tmpBuffer.asReadOnlyBuffer();
success = true;
} finally {
if (success) {
IOUtils.close(is);
} else {
IOUtils.closeWhileHandlingException(is);
}
}
}

10
lucene/analysis/nori/src/test/org/apache/lucene/analysis/ko/StringMockResourceLoader.java
View File

@ -39,16 +39,6 @@ class StringMockResourceLoader implements ResourceLoader {
}
}
@Override
public <T> T newInstance(String cname, Class<T> expectedType) {
Class<? extends T> clazz = findClass(cname, expectedType);
try {
return clazz.getConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("Cannot create instance: " + cname, e);
}
}
@Override
public InputStream openResource(String resource) throws IOException {
return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8));

6
lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/SmartChineseAnalyzer.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.analysis.cn.smart;
import java.io.IOException;
import java.io.UncheckedIOException;
import java.nio.charset.StandardCharsets;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
@ -86,8 +85,9 @@ public final class SmartChineseAnalyzer extends Analyzer {
// make sure it is unmodifiable as we expose it in the outer class
return CharArraySet.unmodifiableSet(
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
SmartChineseAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
IOUtils.requireResourceNonNull(
SmartChineseAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
STOPWORD_FILE_COMMENT));
}
}

6
lucene/analysis/stempel/src/java/org/apache/lucene/analysis/pl/PolishAnalyzer.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.analysis.pl;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharArraySet;
import org.apache.lucene.analysis.LowerCaseFilter;
@ -75,8 +74,9 @@ public final class PolishAnalyzer extends StopwordAnalyzerBase {
try {
DEFAULT_STOP_SET =
WordlistLoader.getWordSet(
IOUtils.getDecodingReader(
PolishAnalyzer.class, DEFAULT_STOPWORD_FILE, StandardCharsets.UTF_8),
IOUtils.requireResourceNonNull(
PolishAnalyzer.class.getResourceAsStream(DEFAULT_STOPWORD_FILE),
DEFAULT_STOPWORD_FILE),
"#");
} catch (IOException ex) {
// default set should always be present as it is part of the

4
lucene/core.tests/src/test/module-info.java
View File

@ -22,4 +22,8 @@ module org.apache.lucene.core.tests {
requires junit;
exports org.apache.lucene.core.tests;
// this is to test ModuleResourceLoader
opens org.apache.lucene.core.testresources to
org.apache.lucene.core;
}

1
lucene/core.tests/src/test/org/apache/lucene/core/testresources/accessible.txt Normal file
View File

@ -0,0 +1 @@
This file should be accessible by ModuleResourceLoader.

68
lucene/core.tests/src/test/org/apache/lucene/core/tests/TestModuleResourceLoader.java Normal file
View File

@ -0,0 +1,68 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.core.tests;
import java.io.IOException;
import org.apache.lucene.util.ModuleResourceLoader;
import org.apache.lucene.util.ResourceLoader;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Test;
public class TestModuleResourceLoader extends Assert {
private static final Module MODULE = TestModuleResourceLoader.class.getModule();
private final ResourceLoader loader = new ModuleResourceLoader(MODULE);
@BeforeClass
public static void beforeClass() {
assertTrue("Test class must be in a named module", MODULE.isNamed());
}
@Test
public void testModuleResources() throws Exception {
try (var stream = loader.openResource("org/apache/lucene/core/testresources/accessible.txt")) {
stream.available();
}
assertNotNull(
"resource should exist when loaded by test from classloader",
getClass().getResource("/org/apache/lucene/core/testresources/accessible.txt"));
try (var stream = loader.openResource("org/apache/lucene/core/tests/nonaccessible.txt")) {
stream.available();
fail("Should throw exception");
} catch (IOException e) {
assertTrue(e.getMessage().startsWith("Resource not found:"));
}
}
@Test
public void testModuleClassloading() throws Exception {
assertSame(
TestModuleResourceLoader.class,
loader.findClass(TestModuleResourceLoader.class.getName(), Object.class));
var cname = "org.foobar.Something";
try {
loader.findClass(cname, Object.class);
fail("Should throw exception");
} catch (RuntimeException e) {
assertEquals("Cannot load class: " + cname, e.getMessage());
}
}
}

1
lucene/core.tests/src/test/org/apache/lucene/core/tests/nonaccessible.txt Normal file
View File

@ -0,0 +1 @@
This file should not be accessible by ModuleResourceLoader.

25
lucene/core/src/java/org/apache/lucene/analysis/StopwordAnalyzerBase.java
View File

@ -71,20 +71,27 @@ public abstract class StopwordAnalyzerBase extends Analyzer {
* @param comment comment string to ignore in the stopword file
* @return a CharArraySet containing the distinct stopwords from the given file
* @throws IOException if loading the stopwords throws an {@link IOException}
* @deprecated {@link Class#getResourceAsStream(String)} is caller sensitive and cannot load
* resources across Java Modules. Please call the {@code getResourceAsStream()} and {@link
* WordlistLoader#getWordSet(Reader, String, CharArraySet)} or other methods directly.
*/
@Deprecated(forRemoval = true, since = "9.1")
protected static CharArraySet loadStopwordSet(
final boolean ignoreCase,
final Class<? extends Analyzer> aClass,
final String resource,
final String comment)
throws IOException {
Reader reader = null;
try {
reader =
IOUtils.getDecodingReader(aClass.getResourceAsStream(resource), StandardCharsets.UTF_8);
var argModule = aClass.getModule();
if (argModule.isNamed() && argModule != StopwordAnalyzerBase.class.getModule()) {
throw new UnsupportedOperationException(
"loadStopwordSet(class,...) does not work when Java Module System is enabled.");
}
try (Reader reader =
IOUtils.getDecodingReader(
IOUtils.requireResourceNonNull(aClass.getResourceAsStream(resource), resource),
StandardCharsets.UTF_8)) {
return WordlistLoader.getWordSet(reader, comment, new CharArraySet(16, ignoreCase));
} finally {
IOUtils.close(reader);
}
}
@ -96,12 +103,8 @@ public abstract class StopwordAnalyzerBase extends Analyzer {
* @throws IOException if loading the stopwords throws an {@link IOException}
*/
protected static CharArraySet loadStopwordSet(Path stopwords) throws IOException {
Reader reader = null;
try {
reader = Files.newBufferedReader(stopwords, StandardCharsets.UTF_8);
try (Reader reader = Files.newBufferedReader(stopwords, StandardCharsets.UTF_8)) {
return WordlistLoader.getWordSet(reader);
} finally {
IOUtils.close(reader);
}
}

114
lucene/core/src/java/org/apache/lucene/analysis/WordlistLoader.java
View File

@ -21,6 +21,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.util.IOUtils;
@ -76,17 +77,30 @@ public class WordlistLoader {
}
/**
* Reads lines from a Reader and adds every non-comment line as an entry to a CharArraySet
* (omitting leading and trailing whitespace). Every line of the Reader should contain only one
* word. The words need to be in lowercase if you make use of an Analyzer which uses
* LowerCaseFilter (like StandardAnalyzer).
* Reads lines from an InputStream with UTF-8 charset and adds every line as an entry to a
* CharArraySet (omitting leading and trailing whitespace). Every line of the Reader should
* contain only one word. The words need to be in lowercase if you make use of an Analyzer which
* uses LowerCaseFilter (like StandardAnalyzer).
*
* @param reader Reader containing the wordlist
* @param comment The string representing a comment.
* @return A CharArraySet with the reader's words
* @param stream InputStream containing the wordlist
* @return A {@link CharArraySet} with the reader's words
*/
public static CharArraySet getWordSet(Reader reader, String comment) throws IOException {
return getWordSet(reader, comment, new CharArraySet(INITIAL_CAPACITY, false));
public static CharArraySet getWordSet(InputStream stream) throws IOException {
return getWordSet(stream, StandardCharsets.UTF_8);
}
/**
* Reads lines from an InputStream with the given charset and adds every line as an entry to a
* CharArraySet (omitting leading and trailing whitespace). Every line of the Reader should
* contain only one word. The words need to be in lowercase if you make use of an Analyzer which
* uses LowerCaseFilter (like StandardAnalyzer).
*
* @param stream InputStream containing the wordlist
* @param charset Charset of the wordlist
* @return A {@link CharArraySet} with the reader's words
*/
public static CharArraySet getWordSet(InputStream stream, Charset charset) throws IOException {
return getWordSet(IOUtils.getDecodingReader(stream, charset));
}
/**
@ -117,6 +131,50 @@ public class WordlistLoader {
return result;
}
/**
* Reads lines from a Reader and adds every non-comment line as an entry to a CharArraySet
* (omitting leading and trailing whitespace). Every line of the Reader should contain only one
* word. The words need to be in lowercase if you make use of an Analyzer which uses
* LowerCaseFilter (like StandardAnalyzer).
*
* @param reader Reader containing the wordlist
* @param comment The string representing a comment.
* @return A CharArraySet with the reader's words
*/
public static CharArraySet getWordSet(Reader reader, String comment) throws IOException {
return getWordSet(reader, comment, new CharArraySet(INITIAL_CAPACITY, false));
}
/**
* Reads lines from an InputStream with UTF-8 charset and adds every non-comment line as an entry
* to a CharArraySet (omitting leading and trailing whitespace). Every line of the Reader should
* contain only one word. The words need to be in lowercase if you make use of an Analyzer which
* uses LowerCaseFilter (like StandardAnalyzer).
*
* @param stream InputStream in UTF-8 encoding containing the wordlist
* @param comment The string representing a comment.
* @return A CharArraySet with the reader's words
*/
public static CharArraySet getWordSet(InputStream stream, String comment) throws IOException {
return getWordSet(stream, StandardCharsets.UTF_8, comment);
}
/**
* Reads lines from an InputStream with the given charset and adds every non-comment line as an
* entry to a CharArraySet (omitting leading and trailing whitespace). Every line of the Reader
* should contain only one word. The words need to be in lowercase if you make use of an Analyzer
* which uses LowerCaseFilter (like StandardAnalyzer).
*
* @param stream InputStream containing the wordlist
* @param charset Charset of the wordlist
* @param comment The string representing a comment.
* @return A CharArraySet with the reader's words
*/
public static CharArraySet getWordSet(InputStream stream, Charset charset, String comment)
throws IOException {
return getWordSet(IOUtils.getDecodingReader(stream, charset), comment);
}
/**
* Reads stopwords from a stopword list in Snowball format.
*
@ -170,6 +228,44 @@ public class WordlistLoader {
return getSnowballWordSet(reader, new CharArraySet(INITIAL_CAPACITY, false));
}
/**
* Reads stopwords from a stopword list in Snowball format.
*
* <p>The snowball format is the following:
*
* <ul>
* <li>Lines may contain multiple words separated by whitespace.
* <li>The comment character is the vertical line (&#124;).
* <li>Lines may contain trailing comments.
* </ul>
*
* @param stream InputStream in UTF-8 encoding containing a Snowball stopword list
* @return A {@link CharArraySet} with the reader's words
*/
public static CharArraySet getSnowballWordSet(InputStream stream) throws IOException {
return getSnowballWordSet(stream, StandardCharsets.UTF_8);
}
/**
* Reads stopwords from a stopword list in Snowball format.
*
* <p>The snowball format is the following:
*
* <ul>
* <li>Lines may contain multiple words separated by whitespace.
* <li>The comment character is the vertical line (&#124;).
* <li>Lines may contain trailing comments.
* </ul>
*
* @param stream InputStream containing a Snowball stopword list
* @param charset Charset of the stopword list
* @return A {@link CharArraySet} with the reader's words
*/
public static CharArraySet getSnowballWordSet(InputStream stream, Charset charset)
throws IOException {
return getSnowballWordSet(IOUtils.getDecodingReader(stream, charset));
}
/**
* Reads a stem dictionary. Each line contains:
*

24
lucene/core/src/java/org/apache/lucene/util/ClasspathResourceLoader.java
View File

@ -22,6 +22,13 @@ import java.io.InputStream;
/**
* Simple {@link ResourceLoader} that uses {@link ClassLoader#getResourceAsStream(String)} and
* {@link Class#forName(String,boolean,ClassLoader)} to open resources and classes, respectively.
*
* <p>To use this class with the Java Module System, you must open all modules on classpath that
* contain resources to the {@code org.apache.lucene.core} module, otherwise resources can't be
* looked up. It is recommended to use {@link ModuleResourceLoader} for such use cases as this would
* limit to certain modules.
*
* @see ModuleResourceLoader
*/
public final class ClasspathResourceLoader implements ResourceLoader {
private final Class<?> clazz;
@ -54,7 +61,12 @@ public final class ClasspathResourceLoader implements ResourceLoader {
(clazz != null)
? clazz.getResourceAsStream(resource)
: loader.getResourceAsStream(resource);
if (stream == null) throw new IOException("Resource not found: " + resource);
if (stream == null) {
throw new IOException(
"Resource not found (if you use Java Module System, make sure to open "
+ "module and package containing resources to 'org.apache.lucene.core' module): "
+ resource);
}
return stream;
}
@ -66,14 +78,4 @@ public final class ClasspathResourceLoader implements ResourceLoader {
throw new RuntimeException("Cannot load class: " + cname, e);
}
}
@Override
public <T> T newInstance(String cname, Class<T> expectedType) {
Class<? extends T> clazz = findClass(cname, expectedType);
try {
return clazz.getConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("Cannot create instance: " + cname, e);
}
}
}

30
lucene/core/src/java/org/apache/lucene/util/IOUtils.java
View File

@ -18,6 +18,7 @@ package org.apache.lucene.util;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@ -156,6 +157,23 @@ public final class IOUtils {
return new BufferedReader(new InputStreamReader(stream, charSetDecoder));
}
/**
* Wrap all calls to {@link Class#getResource(String)} or {@link
* Class#getResourceAsStream(String)} using this method to enforce existence of the resource. This
* code works around those methods returning {@code null} to signal non-existence.
*
* @param resource return value of above methods
* @param name of resource
* @return the resource passed in if existent
* @throws FileNotFoundException if resource was not found
*/
public static <T> T requireResourceNonNull(T resource, String name) throws IOException {
if (resource == null) {
throw new FileNotFoundException("The resource '" + name + "' was not found.");
}
return resource;
}
/**
* Opens a Reader for the given resource using a {@link CharsetDecoder}. Unlike Java's defaults
* this reader will throw an exception if your it detects the read charset doesn't match the
@ -168,13 +186,21 @@ public final class IOUtils {
* @param resource the resource name to load
* @param charSet the expected charset
* @return a reader to read the given file
* @deprecated {@link Class#getResourceAsStream(String)} is caller sensitive and cannot load
* resources across Java Modules. Please call the {@code getResourceAsStream()} directly and
* use {@link #requireResourceNonNull(Object,String)} to signal missing resources {@code null}
*/
@Deprecated(forRemoval = true, since = "9.1")
public static Reader getDecodingReader(Class<?> clazz, String resource, Charset charSet)
throws IOException {
InputStream stream = null;
var argModule = clazz.getModule();
if (argModule.isNamed() && argModule != IOUtils.class.getModule()) {
throw new UnsupportedOperationException(
"getDecodingReader(class,...) does not work when Java Module System is enabled.");
}
InputStream stream = requireResourceNonNull(clazz.getResourceAsStream(resource), resource);
boolean success = false;
try {
stream = clazz.getResourceAsStream(resource);
final Reader reader = getDecodingReader(stream, charSet);
success = true;
return reader;

55
lucene/core/src/java/org/apache/lucene/util/ModuleResourceLoader.java Normal file
View File

@ -0,0 +1,55 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.lucene.util;
import java.io.IOException;
import java.io.InputStream;
/**
* Simple {@link ResourceLoader} that uses {@link Module#getResourceAsStream(String)} and {@link
* Class#forName(Module,String)} to open resources and classes, respectively. Resource paths must be
* absolute to module's root.
*
* <p>To use this class, you must open the module to the {@code org.apache.lucene.core} module,
* otherwise resources can't be looked up.
*/
public final class ModuleResourceLoader implements ResourceLoader {
private final Module module;
/** Creates an instance using the given Java Module to load resources and classes. */
public ModuleResourceLoader(Module module) {
this.module = module;
}
@Override
public InputStream openResource(String resource) throws IOException {
final var stream = module.getResourceAsStream(resource);
if (stream == null) throw new IOException("Resource not found: " + resource);
return stream;
}
@Override
public <T> Class<? extends T> findClass(String cname, Class<T> expectedType) {
try {
final var clazz = Class.forName(module, cname);
if (clazz == null) throw new ClassNotFoundException(cname);
return clazz.asSubclass(expectedType);
} catch (Exception e) {
throw new RuntimeException("Cannot load class: " + cname, e);
}
}
}

9
lucene/core/src/java/org/apache/lucene/util/ResourceLoader.java
View File

@ -30,5 +30,12 @@ public interface ResourceLoader {
/** Creates an instance of the name and expected type */
// TODO: fix exception handling
public <T> T newInstance(String cname, Class<T> expectedType);
public default <T> T newInstance(String cname, Class<T> expectedType) {
Class<? extends T> clazz = findClass(cname, expectedType);
try {
return clazz.getConstructor().newInstance();
} catch (Exception e) {
throw new RuntimeException("Cannot create instance: " + cname, e);
}
}
}

5
lucene/expressions/src/java/org/apache/lucene/expressions/js/JavascriptCompiler.java
View File

@ -730,10 +730,11 @@ public final class JavascriptCompiler {
Map<String, Method> map = new HashMap<>();
try {
final Properties props = new Properties();
var name = JavascriptCompiler.class.getSimpleName() + ".properties";
try (Reader in =
IOUtils.getDecodingReader(
JavascriptCompiler.class,
JavascriptCompiler.class.getSimpleName() + ".properties",
IOUtils.requireResourceNonNull(
JavascriptCompiler.class.getResourceAsStream(name), name),
StandardCharsets.UTF_8)) {
props.load(in);
}

6
lucene/suggest/src/test/org/apache/lucene/search/suggest/TestLookupBenchmark.java
View File

@ -43,6 +43,7 @@ import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.analysis.MockTokenizer;
import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.util.IOUtils;
import org.junit.BeforeClass;
import org.junit.Ignore;
@ -91,8 +92,9 @@ public class TestLookupBenchmark extends LuceneTestCase {
/** Collect the multilingual input for benchmarks/ tests. */
public static List<Input> readTop50KWiki() throws Exception {
List<Input> input = new ArrayList<>();
URL resource = TestLookupBenchmark.class.getResource("Top50KWiki.utf8");
assert resource != null : "Resource missing: Top50KWiki.utf8";
var name = "Top50KWiki.utf8";
URL resource =
IOUtils.requireResourceNonNull(TestLookupBenchmark.class.getResource(name), name);
String line = null;
BufferedReader br = new BufferedReader(new InputStreamReader(resource.openStream(), UTF_8));

15
lucene/test-framework/src/java/org/apache/lucene/tests/util/LuceneTestCase.java
View File

@ -62,6 +62,7 @@ import java.lang.annotation.Target;
import java.lang.reflect.Constructor;
import java.lang.reflect.Method;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileSystem;
import java.nio.file.NoSuchFileException;
@ -151,6 +152,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CommandLineUtil;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.NamedThreadFactory;
import org.apache.lucene.util.SuppressForbidden;
@ -2079,19 +2081,16 @@ public abstract class LuceneTestCase extends Assert {
*/
protected Path getDataPath(String name) throws IOException {
try {
return Paths.get(this.getClass().getResource(name).toURI());
} catch (Exception e) {
throw new IOException("Cannot find resource: " + name, e);
return Paths.get(
IOUtils.requireResourceNonNull(this.getClass().getResource(name), name).toURI());
} catch (URISyntaxException e) {
throw new AssertionError(e);
}
}
/** Gets a resource from the test's classpath as {@link InputStream}. */
protected InputStream getDataInputStream(String name) throws IOException {
InputStream in = this.getClass().getResourceAsStream(name);
if (in == null) {
throw new IOException("Cannot find resource: " + name);
}
return in;
return IOUtils.requireResourceNonNull(this.getClass().getResourceAsStream(name), name);
}
public void assertReaderEquals(String info, IndexReader leftReader, IndexReader rightReader)