Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-5945: Full cutover to Path api from java.io.File 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1624784 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Robert Muir 2014-09-13 21:46:29 +00:00
parent dca2db73a4
commit 3eb66fb19c
299 changed files with 1659 additions and 1951 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/CHANGES.txt
View File

@ -7,6 +7,8 @@ http://s.apache.org/luceneversions
New Features
* LUCENE-5945: All file handling converted to NIO.2 apis. (Robert Muir)
* SOLR-3359: Added analyzer attribute/property to SynonymFilterFactory.
(Ryo Onodera via Koji Sekiguchi)

17
lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilterFactory.java
View File

@ -17,7 +17,6 @@ package org.apache.lucene.analysis.charfilter;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
@ -59,21 +58,15 @@ public class MappingCharFilterFactory extends CharFilterFactory implements
}
}
// TODO: this should use inputstreams from the loader, not File!
@Override
public void inform(ResourceLoader loader) throws IOException {
if (mapping != null) {
List<String> wlist = null;
File mappingFile = new File(mapping);
if (mappingFile.exists()) {
wlist = getLines(loader, mapping);
} else {
List<String> files = splitFileNames(mapping);
wlist = new ArrayList<>();
for (String file : files) {
List<String> lines = getLines(loader, file.trim());
wlist.addAll(lines);
}
List<String> files = splitFileNames(mapping);
wlist = new ArrayList<>();
for (String file : files) {
List<String> lines = getLines(loader, file.trim());
wlist.addAll(lines);
}
final NormalizeCharMap.Builder builder = new NormalizeCharMap.Builder();
parseRules(wlist, builder);

12
lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
View File

@ -119,18 +119,6 @@ public class HyphenationCompoundWordTokenFilter extends
return getHyphenationTree(new InputSource(hyphenationFilename));
}
/**
* Create a hyphenator tree
*
* @param hyphenationFile the file of the XML grammar to load
* @return An object representing the hyphenation patterns
* @throws java.io.IOException If there is a low-level I/O error.
*/
public static HyphenationTree getHyphenationTree(File hyphenationFile)
throws IOException {
return getHyphenationTree(new InputSource(hyphenationFile.toURI().toASCIIString()));
}
/**
* Create a hyphenator tree
*

13
lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/Lucene43HyphenationCompoundWordTokenFilter.java
View File

@ -17,7 +17,6 @@ package org.apache.lucene.analysis.compound;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
@ -120,18 +119,6 @@ public class Lucene43HyphenationCompoundWordTokenFilter extends
return getHyphenationTree(new InputSource(hyphenationFilename));
}
/**
* Create a hyphenator tree
*
* @param hyphenationFile the file of the XML grammar to load
* @return An object representing the hyphenation patterns
* @throws IOException If there is a low-level I/O error.
*/
public static HyphenationTree getHyphenationTree(File hyphenationFile)
throws IOException {
return getHyphenationTree(new InputSource(hyphenationFile.toURI().toASCIIString()));
}
/**
* Create a hyphenator tree
*

12
lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
View File

@ -17,7 +17,6 @@
package org.apache.lucene.analysis.compound.hyphenation;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.ArrayList;
@ -104,17 +103,6 @@ public class HyphenationTree extends TernaryTree implements PatternConsumer {
return buf.toString();
}
/**
* Read hyphenation patterns from an XML file.
*
* @param f the filename
* @throws IOException In case the parsing fails
*/
public void loadPatterns(File f) throws IOException {
InputSource src = new InputSource(f.toURI().toASCIIString());
loadPatterns(src);
}
/**
* Read hyphenation patterns from an XML file.
*

12
lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
View File

@ -26,7 +26,6 @@ import org.xml.sax.helpers.DefaultHandler;
import org.xml.sax.Attributes;
// Java
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
@ -91,17 +90,6 @@ public class PatternParser extends DefaultHandler {
parse(new InputSource(filename));
}
/**
* Parses a hyphenation pattern file.
*
* @param file the pattern file
* @throws IOException In case of an exception while parsing
*/
public void parse(File file) throws IOException {
InputSource src = new InputSource(file.toURI().toASCIIString());
parse(src);
}
/**
* Parses a hyphenation pattern file.
*

5
lucene/analysis/common/src/java/org/apache/lucene/analysis/core/StopAnalyzer.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.analysis.core;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Path;
import java.util.Arrays;
import java.util.List;
@ -62,10 +63,10 @@ public final class StopAnalyzer extends StopwordAnalyzerBase {
super(stopWords);
}
/** Builds an analyzer with the stop words from the given file.
/** Builds an analyzer with the stop words from the given path.
* @see WordlistLoader#getWordSet(Reader)
* @param stopwordsFile File to load stop words from */
public StopAnalyzer(File stopwordsFile) throws IOException {
public StopAnalyzer(Path stopwordsFile) throws IOException {
this(loadStopwordSet(stopwordsFile));
}

24
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/Dictionary.java
View File

@ -42,9 +42,6 @@ import org.apache.lucene.util.fst.Util;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@ -55,6 +52,7 @@ import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.ParseException;
import java.util.ArrayList;
import java.util.Arrays;
@ -141,7 +139,7 @@ public class Dictionary {
// when set, some words have exceptional stems, and the last entry is a pointer to stemExceptions
boolean hasStemExceptions;
private final File tempDir = OfflineSorter.defaultTempDir(); // TODO: make this configurable?
private final Path tempDir = OfflineSorter.defaultTempDir(); // TODO: make this configurable?
boolean ignoreCase;
boolean complexPrefixes;
@ -200,8 +198,8 @@ public class Dictionary {
this.needsOutputCleaning = false; // set if we have an OCONV
flagLookup.add(new BytesRef()); // no flags -> ord 0
File aff = File.createTempFile("affix", "aff", tempDir);
OutputStream out = new BufferedOutputStream(new FileOutputStream(aff));
Path aff = Files.createTempFile(tempDir, "affix", "aff");
OutputStream out = new BufferedOutputStream(Files.newOutputStream(aff));
InputStream aff1 = null;
InputStream aff2 = null;
boolean success = false;
@ -215,12 +213,12 @@ public class Dictionary {
out.close();
// pass 1: get encoding
aff1 = new BufferedInputStream(new FileInputStream(aff));
aff1 = new BufferedInputStream(Files.newInputStream(aff));
String encoding = getDictionaryEncoding(aff1);
// pass 2: parse affixes
CharsetDecoder decoder = getJavaEncoding(encoding);
aff2 = new BufferedInputStream(new FileInputStream(aff));
aff2 = new BufferedInputStream(Files.newInputStream(aff));
readAffixFile(aff2, decoder);
// read dictionary entries
@ -234,7 +232,7 @@ public class Dictionary {
} finally {
IOUtils.closeWhileHandlingException(out, aff1, aff2);
if (success) {
Files.delete(aff.toPath());
Files.delete(aff);
} else {
IOUtils.deleteFilesIgnoringExceptions(aff);
}
@ -782,7 +780,7 @@ public class Dictionary {
StringBuilder sb = new StringBuilder();
File unsorted = File.createTempFile("unsorted", "dat", tempDir);
Path unsorted = Files.createTempFile(tempDir, "unsorted", "dat");
try (ByteSequencesWriter writer = new ByteSequencesWriter(unsorted)) {
for (InputStream dictionary : dictionaries) {
BufferedReader lines = new BufferedReader(new InputStreamReader(dictionary, decoder));
@ -825,7 +823,7 @@ public class Dictionary {
}
}
}
File sorted = File.createTempFile("sorted", "dat", tempDir);
Path sorted = Files.createTempFile(tempDir, "sorted", "dat");
OfflineSorter sorter = new OfflineSorter(new Comparator<BytesRef>() {
BytesRef scratch1 = new BytesRef();
@ -870,7 +868,7 @@ public class Dictionary {
success = true;
} finally {
if (success) {
Files.delete(unsorted.toPath());
Files.delete(unsorted);
} else {
IOUtils.deleteFilesIgnoringExceptions(unsorted);
}
@ -960,7 +958,7 @@ public class Dictionary {
} finally {
IOUtils.closeWhileHandlingException(reader);
if (success2) {
Files.delete(sorted.toPath());
Files.delete(sorted);
} else {
IOUtils.deleteFilesIgnoringExceptions(sorted);
}

13
lucene/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilterFactory.java
View File

@ -17,7 +17,6 @@ package org.apache.lucene.analysis.synonym;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
@ -171,16 +170,10 @@ public class SynonymFilterFactory extends TokenFilterFactory implements Resource
throw new RuntimeException(e);
}
File synonymFile = new File(synonyms);
if (synonymFile.exists()) {
List<String> files = splitFileNames(synonyms);
for (String file : files) {
decoder.reset();
parser.parse(new InputStreamReader(loader.openResource(synonyms), decoder));
} else {
List<String> files = splitFileNames(synonyms);
for (String file : files) {
decoder.reset();
parser.parse(new InputStreamReader(loader.openResource(file), decoder));
}
parser.parse(new InputStreamReader(loader.openResource(file), decoder));
}
return parser.build();
}

35
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/FilesystemResourceLoader.java
View File

@ -17,11 +17,12 @@ package org.apache.lucene.analysis.util;
* limitations under the License.
*/
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.NoSuchFileException;
import java.nio.file.Path;
/**
* Simple {@link ResourceLoader} that opens resource files
@ -37,25 +38,16 @@ import java.io.InputStream;
* to allow lookup of files in more than one base directory.
*/
public final class FilesystemResourceLoader implements ResourceLoader {
private final File baseDirectory;
private final Path baseDirectory;
private final ResourceLoader delegate;
/**
* Creates a resource loader that requires absolute filenames or relative to CWD
* to resolve resources. Files not found in file system and class lookups
* are delegated to context classloader.
*/
public FilesystemResourceLoader() {
this((File) null);
}
/**
* Creates a resource loader that resolves resources against the given
* base directory (may be {@code null} to refer to CWD).
* Files not found in file system and class lookups are delegated to context
* classloader.
*/
public FilesystemResourceLoader(File baseDirectory) {
public FilesystemResourceLoader(Path baseDirectory) {
this(baseDirectory, new ClasspathResourceLoader());
}
@ -65,9 +57,12 @@ public final class FilesystemResourceLoader implements ResourceLoader {
* Files not found in file system and class lookups are delegated
* to the given delegate {@link ResourceLoader}.
*/
public FilesystemResourceLoader(File baseDirectory, ResourceLoader delegate) {
if (baseDirectory != null && !baseDirectory.isDirectory())
throw new IllegalArgumentException("baseDirectory is not a directory or null");
public FilesystemResourceLoader(Path baseDirectory, ResourceLoader delegate) {
if (baseDirectory == null) {
throw new NullPointerException();
}
if (!Files.isDirectory(baseDirectory))
throw new IllegalArgumentException(baseDirectory + " is not a directory");
if (delegate == null)
throw new IllegalArgumentException("delegate ResourceLoader may not be null");
this.baseDirectory = baseDirectory;
@ -77,12 +72,8 @@ public final class FilesystemResourceLoader implements ResourceLoader {
@Override
public InputStream openResource(String resource) throws IOException {
try {
File file = new File (resource);
if (baseDirectory != null && !file.isAbsolute()) {
file = new File(baseDirectory, resource);
}
return new FileInputStream(file);
} catch (FileNotFoundException fnfe) {
return Files.newInputStream(baseDirectory.resolve(resource));
} catch (FileNotFoundException | NoSuchFileException fnfe) {
return delegate.openResource(resource);
}
}

9
lucene/analysis/common/src/java/org/apache/lucene/analysis/util/StopwordAnalyzerBase.java
View File

@ -17,10 +17,11 @@
package org.apache.lucene.analysis.util;
import java.io.File;
import java.io.IOException;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.util.IOUtils;
@ -98,7 +99,7 @@ public abstract class StopwordAnalyzerBase extends Analyzer {
}
/**
* Creates a CharArraySet from a file.
* Creates a CharArraySet from a path.
*
* @param stopwords
* the stopwords file to load
@ -107,10 +108,10 @@ public abstract class StopwordAnalyzerBase extends Analyzer {
* @throws IOException
* if loading the stopwords throws an {@link IOException}
*/
protected static CharArraySet loadStopwordSet(File stopwords) throws IOException {
protected static CharArraySet loadStopwordSet(Path stopwords) throws IOException {
Reader reader = null;
try {
reader = IOUtils.getDecodingReader(stopwords, StandardCharsets.UTF_8);
reader = Files.newBufferedReader(stopwords, StandardCharsets.UTF_8);
return WordlistLoader.getWordSet(reader);
} finally {
IOUtils.close(reader);

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/ckb/TestSoraniStemFilter.java
View File

@ -95,6 +95,6 @@ public class TestSoraniStemFilter extends BaseTokenStreamTestCase {
/** test against a basic vocabulary file */
public void testVocabulary() throws Exception {
// top 8k words or so: freq > 1000
assertVocabulary(a, getDataFile("ckbtestdata.zip"), "testdata.txt");
assertVocabulary(a, getDataPath("ckbtestdata.zip"), "testdata.txt");
}
}

41
lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestRandomChains.java
View File

@ -17,7 +17,6 @@ package org.apache.lucene.analysis.core;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.Reader;
@ -28,6 +27,10 @@ import java.lang.reflect.Modifier;
import java.net.URI;
import java.net.URL;
import java.nio.CharBuffer;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -271,23 +274,25 @@ public class TestRandomChains extends BaseTokenStreamTestCase {
final URI uri = resources.nextElement().toURI();
if (!"file".equalsIgnoreCase(uri.getScheme()))
continue;
final File directory = new File(uri);
if (directory.exists()) {
String[] files = directory.list();
for (String file : files) {
if (new File(directory, file).isDirectory()) {
// recurse
String subPackage = pckgname + "." + file;
collectClassesForPackage(subPackage, classes);
}
if (file.endsWith(".class")) {
String clazzName = file.substring(0, file.length() - 6);
// exclude Test classes that happen to be in these packages.
// class.ForName'ing some of them can cause trouble.
if (!clazzName.endsWith("Test") && !clazzName.startsWith("Test")) {
// Don't run static initializers, as we won't use most of them.
// Java will do that automatically once accessed/instantiated.
classes.add(Class.forName(pckgname + '.' + clazzName, false, cld));
final Path directory = Paths.get(uri);
if (Files.exists(directory)) {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(directory)) {
for (Path file : stream) {
if (Files.isDirectory(file)) {
// recurse
String subPackage = pckgname + "." + file.getFileName().toString();
collectClassesForPackage(subPackage, classes);
}
String fname = file.getFileName().toString();
if (fname.endsWith(".class")) {
String clazzName = fname.substring(0, fname.length() - 6);
// exclude Test classes that happen to be in these packages.
// class.ForName'ing some of them can cause trouble.
if (!clazzName.endsWith("Test") && !clazzName.startsWith("Test")) {
// Don't run static initializers, as we won't use most of them.
// Java will do that automatically once accessed/instantiated.
classes.add(Class.forName(pckgname + '.' + clazzName, false, cld));
}
}
}
}

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanLightStemFilter.java
View File

@ -45,7 +45,7 @@ public class TestGermanLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("delighttestdata.zip"), "delight.txt");
assertVocabulary(analyzer, getDataPath("delighttestdata.zip"), "delight.txt");
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/de/TestGermanMinimalStemFilter.java
View File

@ -70,7 +70,7 @@ public class TestGermanMinimalStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("deminimaltestdata.zip"), "deminimal.txt");
assertVocabulary(analyzer, getDataPath("deminimaltestdata.zip"), "deminimal.txt");
}
/** blast some random strings through the analyzer */

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestKStemmer.java
View File

@ -51,7 +51,7 @@ public class TestKStemmer extends BaseTokenStreamTestCase {
* testCreateMap, commented out below).
*/
public void testVocabulary() throws Exception {
assertVocabulary(a, getDataFile("kstemTestData.zip"), "kstem_examples.txt");
assertVocabulary(a, getDataPath("kstemTestData.zip"), "kstem_examples.txt");
}
public void testEmptyTerm() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/en/TestPorterStemFilter.java
View File

@ -49,7 +49,7 @@ public class TestPorterStemFilter extends BaseTokenStreamTestCase {
* The output should be the same as the string in output.txt
*/
public void testPorterStemFilter() throws Exception {
assertVocabulary(a, getDataFile("porterTestData.zip"), "voc.txt", "output.txt");
assertVocabulary(a, getDataPath("porterTestData.zip"), "voc.txt", "output.txt");
}
public void testWithKeywordAttribute() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/es/TestSpanishLightStemFilter.java
View File

@ -42,7 +42,7 @@ public class TestSpanishLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("eslighttestdata.zip"), "eslight.txt");
assertVocabulary(analyzer, getDataPath("eslighttestdata.zip"), "eslight.txt");
}
/** blast some random strings through the analyzer */

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/fi/TestFinnishLightStemFilter.java
View File

@ -44,7 +44,7 @@ public class TestFinnishLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("filighttestdata.zip"), "filight.txt");
assertVocabulary(analyzer, getDataPath("filighttestdata.zip"), "filight.txt");
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchLightStemFilter.java
View File

@ -175,7 +175,7 @@ public class TestFrenchLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("frlighttestdata.zip"), "frlight.txt");
assertVocabulary(analyzer, getDataPath("frlighttestdata.zip"), "frlight.txt");
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/fr/TestFrenchMinimalStemFilter.java
View File

@ -72,7 +72,7 @@ public class TestFrenchMinimalStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("frminimaltestdata.zip"), "frminimal.txt");
assertVocabulary(analyzer, getDataPath("frminimaltestdata.zip"), "frminimal.txt");
}
/** blast some random strings through the analyzer */

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/gl/TestGalicianStemFilter.java
View File

@ -46,7 +46,7 @@ public class TestGalicianStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("gltestdata.zip"), "gl.txt");
assertVocabulary(analyzer, getDataPath("gltestdata.zip"), "gl.txt");
}
public void testEmptyTerm() throws IOException {

3
lucene/analysis/common/src/test/org/apache/lucene/analysis/hu/TestHungarianLightStemFilter.java
View File

@ -18,7 +18,6 @@ package org.apache.lucene.analysis.hu;
*/
import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
@ -45,7 +44,7 @@ public class TestHungarianLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("hulighttestdata.zip"), "hulight.txt");
assertVocabulary(analyzer, getDataPath("hulighttestdata.zip"), "hulight.txt");
}
public void testKeyword() throws IOException {

22
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/Test64kAffixes.java
View File

@ -18,12 +18,10 @@ package org.apache.lucene.analysis.hunspell;
*/
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.InputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import org.apache.lucene.util.CharsRef;
@ -33,13 +31,11 @@ import org.apache.lucene.util.LuceneTestCase;
public class Test64kAffixes extends LuceneTestCase {
public void test() throws Exception {
File tempDir = createTempDir("64kaffixes");
File affix = new File(tempDir, "64kaffixes.aff");
File dict = new File(tempDir, "64kaffixes.dic");
Path tempDir = createTempDir("64kaffixes");
Path affix = tempDir.resolve("64kaffixes.aff");
Path dict = tempDir.resolve("64kaffixes.dic");
BufferedWriter affixWriter = new BufferedWriter(
new OutputStreamWriter(
new FileOutputStream(affix), StandardCharsets.UTF_8));
BufferedWriter affixWriter = Files.newBufferedWriter(affix, StandardCharsets.UTF_8);
// 65k affixes with flag 1, then an affix with flag 2
affixWriter.write("SET UTF-8\nFLAG num\nSFX 1 Y 65536\n");
@ -49,15 +45,13 @@ public class Test64kAffixes extends LuceneTestCase {
affixWriter.write("SFX 2 Y 1\nSFX 2 0 s\n");
affixWriter.close();
BufferedWriter dictWriter = new BufferedWriter(
new OutputStreamWriter(
new FileOutputStream(dict), StandardCharsets.UTF_8));
BufferedWriter dictWriter = Files.newBufferedWriter(dict, StandardCharsets.UTF_8);
// drink signed with affix 2 (takes -s)
dictWriter.write("1\ndrink/2\n");
dictWriter.close();
try (InputStream affStream = new FileInputStream(affix); InputStream dictStream = new FileInputStream(dict)) {
try (InputStream affStream = Files.newInputStream(affix); InputStream dictStream = Files.newInputStream(dict)) {
Dictionary dictionary = new Dictionary(affStream, dictStream);
Stemmer stemmer = new Stemmer(dictionary);
// drinks should still stem to drink

20
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries.java
View File

@ -17,9 +17,11 @@ package org.apache.lucene.analysis.hunspell;
* limitations under the License.
*/
import java.io.File;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@ -39,8 +41,8 @@ import org.junit.Ignore;
public class TestAllDictionaries extends LuceneTestCase {
// set this to the location of where you downloaded all the files
static final File DICTIONARY_HOME =
new File("/data/archive.services.openoffice.org/pub/mirror/OpenOffice.org/contrib/dictionaries");
static final Path DICTIONARY_HOME =
Paths.get("/data/archive.services.openoffice.org/pub/mirror/OpenOffice.org/contrib/dictionaries");
final String tests[] = {
/* zip file */ /* dictionary */ /* affix */
@ -156,10 +158,10 @@ public class TestAllDictionaries extends LuceneTestCase {
public void test() throws Exception {
for (int i = 0; i < tests.length; i += 3) {
File f = new File(DICTIONARY_HOME, tests[i]);
assert f.exists();
Path f = DICTIONARY_HOME.resolve(tests[i]);
assert Files.exists(f);
try (ZipFile zip = new ZipFile(f, StandardCharsets.UTF_8)) {
try (ZipFile zip = new ZipFile(f.toFile(), StandardCharsets.UTF_8)) {
ZipEntry dicEntry = zip.getEntry(tests[i+1]);
assert dicEntry != null;
ZipEntry affEntry = zip.getEntry(tests[i+2]);
@ -185,10 +187,10 @@ public class TestAllDictionaries extends LuceneTestCase {
String toTest = "zu_ZA.zip";
for (int i = 0; i < tests.length; i++) {
if (tests[i].equals(toTest)) {
File f = new File(DICTIONARY_HOME, tests[i]);
assert f.exists();
Path f = DICTIONARY_HOME.resolve(tests[i]);
assert Files.exists(f);
try (ZipFile zip = new ZipFile(f, StandardCharsets.UTF_8)) {
try (ZipFile zip = new ZipFile(f.toFile(), StandardCharsets.UTF_8)) {
ZipEntry dicEntry = zip.getEntry(tests[i+1]);
assert dicEntry != null;
ZipEntry affEntry = zip.getEntry(tests[i+2]);

19
lucene/analysis/common/src/test/org/apache/lucene/analysis/hunspell/TestAllDictionaries2.java
View File

@ -17,9 +17,11 @@ package org.apache.lucene.analysis.hunspell;
* limitations under the License.
*/
import java.io.File;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;
@ -39,8 +41,7 @@ import org.junit.Ignore;
public class TestAllDictionaries2 extends LuceneTestCase {
// set this to the location of where you downloaded all the files
static final File DICTIONARY_HOME =
new File("/data/thunderbirdDicts");
static final Path DICTIONARY_HOME = Paths.get("/data/thunderbirdDicts");
final String tests[] = {
/* zip file */ /* dictionary */ /* affix */
@ -172,10 +173,10 @@ public class TestAllDictionaries2 extends LuceneTestCase {
public void test() throws Exception {
for (int i = 0; i < tests.length; i += 3) {
File f = new File(DICTIONARY_HOME, tests[i]);
assert f.exists();
Path f = DICTIONARY_HOME.resolve(tests[i]);
assert Files.exists(f);
try (ZipFile zip = new ZipFile(f, StandardCharsets.UTF_8)) {
try (ZipFile zip = new ZipFile(f.toFile(), StandardCharsets.UTF_8)) {
ZipEntry dicEntry = zip.getEntry(tests[i+1]);
assert dicEntry != null;
ZipEntry affEntry = zip.getEntry(tests[i+2]);
@ -201,10 +202,10 @@ public class TestAllDictionaries2 extends LuceneTestCase {
String toTest = "hungarian_dictionary-1.6.1.1-fx+tb+sm+fn.xpi";
for (int i = 0; i < tests.length; i++) {
if (tests[i].equals(toTest)) {
File f = new File(DICTIONARY_HOME, tests[i]);
assert f.exists();
Path f = DICTIONARY_HOME.resolve(tests[i]);
assert Files.exists(f);
try (ZipFile zip = new ZipFile(f, StandardCharsets.UTF_8)) {
try (ZipFile zip = new ZipFile(f.toFile(), StandardCharsets.UTF_8)) {
ZipEntry dicEntry = zip.getEntry(tests[i+1]);
assert dicEntry != null;
ZipEntry affEntry = zip.getEntry(tests[i+2]);

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/it/TestItalianLightStemFilter.java
View File

@ -42,7 +42,7 @@ public class TestItalianLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("itlighttestdata.zip"), "itlight.txt");
assertVocabulary(analyzer, getDataPath("itlighttestdata.zip"), "itlight.txt");
}
/** blast some random strings through the analyzer */

5
lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianLightStemFilter.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.analysis.no;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@ -50,7 +51,7 @@ public class TestNorwegianLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary file */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_light.txt")));
assertVocabulary(analyzer, Files.newInputStream(getDataPath("nb_light.txt")));
}
/** Test against a Nynorsk vocabulary file */
@ -62,7 +63,7 @@ public class TestNorwegianLightStemFilter extends BaseTokenStreamTestCase {
return new TokenStreamComponents(source, new NorwegianLightStemFilter(source, NYNORSK));
}
};
assertVocabulary(analyzer, new FileInputStream(getDataFile("nn_light.txt")));
assertVocabulary(analyzer, Files.newInputStream(getDataPath("nn_light.txt")));
}
public void testKeyword() throws IOException {

5
lucene/analysis/common/src/test/org/apache/lucene/analysis/no/TestNorwegianMinimalStemFilter.java
View File

@ -20,6 +20,7 @@ package org.apache.lucene.analysis.no;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.Reader;
import java.nio.file.Files;
import java.util.Random;
import org.apache.lucene.analysis.Analyzer;
@ -49,7 +50,7 @@ public class TestNorwegianMinimalStemFilter extends BaseTokenStreamTestCase {
/** Test against a Bokmål vocabulary file */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, new FileInputStream(getDataFile("nb_minimal.txt")));
assertVocabulary(analyzer, Files.newInputStream(getDataPath("nb_minimal.txt")));
}
/** Test against a Nynorsk vocabulary file */
@ -61,7 +62,7 @@ public class TestNorwegianMinimalStemFilter extends BaseTokenStreamTestCase {
return new TokenStreamComponents(source, new NorwegianMinimalStemFilter(source, NYNORSK));
}
};
assertVocabulary(analyzer, new FileInputStream(getDataFile("nn_minimal.txt")));
assertVocabulary(analyzer, Files.newInputStream(getDataPath("nn_minimal.txt")));
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseLightStemFilter.java
View File

@ -88,7 +88,7 @@ public class TestPortugueseLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("ptlighttestdata.zip"), "ptlight.txt");
assertVocabulary(analyzer, getDataPath("ptlighttestdata.zip"), "ptlight.txt");
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseMinimalStemFilter.java
View File

@ -62,7 +62,7 @@ public class TestPortugueseMinimalStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("ptminimaltestdata.zip"), "ptminimal.txt");
assertVocabulary(analyzer, getDataPath("ptminimaltestdata.zip"), "ptminimal.txt");
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/pt/TestPortugueseStemFilter.java
View File

@ -62,7 +62,7 @@ public class TestPortugueseStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("ptrslptestdata.zip"), "ptrslp.txt");
assertVocabulary(analyzer, getDataPath("ptrslptestdata.zip"), "ptrslp.txt");
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/ru/TestRussianLightStemFilter.java
View File

@ -45,7 +45,7 @@ public class TestRussianLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("rulighttestdata.zip"), "rulight.txt");
assertVocabulary(analyzer, getDataPath("rulighttestdata.zip"), "rulight.txt");
}
public void testKeyword() throws IOException {

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/snowball/TestSnowballVocab.java
View File

@ -76,7 +76,7 @@ public class TestSnowballVocab extends LuceneTestCase {
}
};
assertVocabulary(a, getDataFile("TestSnowballVocabData.zip"),
assertVocabulary(a, getDataPath("TestSnowballVocabData.zip"),
dataDirectory + "/voc.txt", dataDirectory + "/output.txt");
}
}

2
lucene/analysis/common/src/test/org/apache/lucene/analysis/sv/TestSwedishLightStemFilter.java
View File

@ -45,7 +45,7 @@ public class TestSwedishLightStemFilter extends BaseTokenStreamTestCase {
/** Test against a vocabulary from the reference impl */
public void testVocabulary() throws IOException {
assertVocabulary(analyzer, getDataFile("svlighttestdata.zip"), "svlight.txt");
assertVocabulary(analyzer, getDataPath("svlighttestdata.zip"), "svlight.txt");
}
public void testKeyword() throws IOException {

18
lucene/analysis/common/src/test/org/apache/lucene/analysis/util/TestFilesystemResourceLoader.java
View File

@ -24,6 +24,8 @@ import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
@ -61,10 +63,9 @@ public class TestFilesystemResourceLoader extends LuceneTestCase {
}
public void testBaseDir() throws Exception {
final File base = createTempDir("fsResourceLoaderBase").getAbsoluteFile();
final Path base = createTempDir("fsResourceLoaderBase");
try {
base.mkdirs();
Writer os = new OutputStreamWriter(new FileOutputStream(new File(base, "template.txt")), StandardCharsets.UTF_8);
Writer os = Files.newBufferedWriter(base.resolve("template.txt"), StandardCharsets.UTF_8);
try {
os.write("foobar\n");
} finally {
@ -74,25 +75,18 @@ public class TestFilesystemResourceLoader extends LuceneTestCase {
ResourceLoader rl = new FilesystemResourceLoader(base);
assertEquals("foobar", WordlistLoader.getLines(rl.openResource("template.txt"), StandardCharsets.UTF_8).get(0));
// Same with full path name:
String fullPath = new File(base, "template.txt").toString();
String fullPath = base.resolve("template.txt").toAbsolutePath().toString();
assertEquals("foobar",
WordlistLoader.getLines(rl.openResource(fullPath), StandardCharsets.UTF_8).get(0));
assertClasspathDelegation(rl);
assertNotFound(rl);
// now use RL without base dir:
rl = new FilesystemResourceLoader();
assertEquals("foobar",
WordlistLoader.getLines(rl.openResource(new File(base, "template.txt").toString()), StandardCharsets.UTF_8).get(0));
assertClasspathDelegation(rl);
assertNotFound(rl);
} finally {
IOUtils.rm(base);
}
}
public void testDelegation() throws Exception {
ResourceLoader rl = new FilesystemResourceLoader(null, new StringMockResourceLoader("foobar\n"));
ResourceLoader rl = new FilesystemResourceLoader(createTempDir("empty"), new StringMockResourceLoader("foobar\n"));
assertEquals("foobar", WordlistLoader.getLines(rl.openResource("template.txt"), StandardCharsets.UTF_8).get(0));
}

10
lucene/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/ja/util/TokenInfoDictionaryWriter.java
View File

@ -17,8 +17,10 @@ package org.apache.lucene.analysis.ja.util;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.lucene.analysis.ja.dict.TokenInfoDictionary;
import org.apache.lucene.util.fst.FST;
@ -41,8 +43,8 @@ public class TokenInfoDictionaryWriter extends BinaryDictionaryWriter {
}
protected void writeFST(String filename) throws IOException {
File f = new File(filename);
f.getParentFile().mkdirs();
fst.save(f);
Path p = Paths.get(filename);
Files.createDirectories(p.getParent());
fst.save(p);
}
}

40
lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/AnalyzerProfile.java
View File

@ -17,11 +17,12 @@
package org.apache.lucene.analysis.cn.smart;
import java.io.File;
import java.io.FileInputStream;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
/**
@ -51,16 +52,18 @@ public class AnalyzerProfile {
if (ANALYSIS_DATA_DIR.length() != 0)
return;
File[] cadidateFiles = new File[] { new File("./" + dirName),
new File("./lib/" + dirName), new File("./" + propName),
new File("./lib/" + propName) };
for (int i = 0; i < cadidateFiles.length; i++) {
File file = cadidateFiles[i];
if (file.exists()) {
if (file.isDirectory()) {
ANALYSIS_DATA_DIR = file.getAbsolutePath();
} else if (file.isFile() && getAnalysisDataDir(file).length() != 0) {
ANALYSIS_DATA_DIR = getAnalysisDataDir(file);
Path[] candidateFiles = new Path[] {
Paths.get(dirName),
Paths.get("lib").resolve(dirName),
Paths.get(propName),
Paths.get("lib").resolve(propName)
};
for (Path file : candidateFiles) {
if (Files.exists(file)) {
if (Files.isDirectory(file)) {
ANALYSIS_DATA_DIR = file.toAbsolutePath().toString();
} else if (Files.isRegularFile(file) && getAnalysisDataDir(file).length() != 0) {
ANALYSIS_DATA_DIR = getAnalysisDataDir(file).toString();
}
break;
}
@ -75,14 +78,11 @@ public class AnalyzerProfile {
}
private static String getAnalysisDataDir(File propFile) {
private static String getAnalysisDataDir(Path propFile) {
Properties prop = new Properties();
try {
FileInputStream input = new FileInputStream(propFile);
prop.load(new InputStreamReader(input, StandardCharsets.UTF_8));
String dir = prop.getProperty("analysis.data.dir", "");
input.close();
return dir;
try (BufferedReader reader = Files.newBufferedReader(propFile, StandardCharsets.UTF_8)) {
prop.load(reader);
return prop.getProperty("analysis.data.dir", "");
} catch (IOException e) {
return "";
}

24
lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
View File

@ -17,18 +17,16 @@
package org.apache.lucene.analysis.cn.smart.hhmm;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
@ -75,9 +73,9 @@ class BigramDictionary extends AbstractDictionary {
return singleInstance;
}
private boolean loadFromObj(File serialObj) {
private boolean loadFromObj(Path serialObj) {
try {
loadFromInputStream(new FileInputStream(serialObj));
loadFromInputStream(Files.newInputStream(serialObj));
return true;
} catch (Exception e) {
throw new RuntimeException(e);
@ -93,9 +91,9 @@ class BigramDictionary extends AbstractDictionary {
input.close();
}
private void saveToObj(File serialObj) {
private void saveToObj(Path serialObj) {
try {
ObjectOutputStream output = new ObjectOutputStream(new FileOutputStream(
ObjectOutputStream output = new ObjectOutputStream(Files.newOutputStream(
serialObj));
output.writeObject(bigramHashTable);
output.writeObject(frequencyTable);
@ -114,9 +112,9 @@ class BigramDictionary extends AbstractDictionary {
private void load(String dictRoot) {
String bigramDictPath = dictRoot + "/bigramdict.dct";
File serialObj = new File(dictRoot + "/bigramdict.mem");
Path serialObj = Paths.get(dictRoot + "/bigramdict.mem");
if (serialObj.exists() && loadFromObj(serialObj)) {
if (Files.exists(serialObj) && loadFromObj(serialObj)) {
} else {
try {
@ -149,7 +147,7 @@ class BigramDictionary extends AbstractDictionary {
int[] buffer = new int[3];
byte[] intBuffer = new byte[4];
String tmpword;
RandomAccessFile dctFile = new RandomAccessFile(dctFilePath, "r");
DataInputStream dctFile = new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)));
// GB2312 characters 0 - 6768
for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {

24
lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
View File

@ -17,18 +17,16 @@
package org.apache.lucene.analysis.cn.smart.hhmm;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.DataInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.ObjectInputStream;
import java.io.ObjectOutputStream;
import java.io.RandomAccessFile;
import java.io.UnsupportedEncodingException;
import java.nio.ByteBuffer;
import java.nio.ByteOrder;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.lucene.analysis.cn.smart.AnalyzerProfile;
import org.apache.lucene.analysis.cn.smart.Utility;
@ -101,9 +99,9 @@ class WordDictionary extends AbstractDictionary {
*/
public void load(String dctFileRoot) {
String dctFilePath = dctFileRoot + "/coredict.dct";
File serialObj = new File(dctFileRoot + "/coredict.mem");
Path serialObj = Paths.get(dctFileRoot + "/coredict.mem");
if (serialObj.exists() && loadFromObj(serialObj)) {
if (Files.exists(serialObj) && loadFromObj(serialObj)) {
} else {
try {
@ -140,9 +138,9 @@ class WordDictionary extends AbstractDictionary {
loadFromObjectInputStream(input);
}
private boolean loadFromObj(File serialObj) {
private boolean loadFromObj(Path serialObj) {
try {
loadFromObjectInputStream(new FileInputStream(serialObj));
loadFromObjectInputStream(Files.newInputStream(serialObj));
return true;
} catch (Exception e) {
throw new RuntimeException(e);
@ -160,9 +158,9 @@ class WordDictionary extends AbstractDictionary {
input.close();
}
private void saveToObj(File serialObj) {
private void saveToObj(Path serialObj) {
try {
ObjectOutputStream output = new ObjectOutputStream(new FileOutputStream(
ObjectOutputStream output = new ObjectOutputStream(Files.newOutputStream(
serialObj));
output.writeObject(wordIndexTable);
output.writeObject(charIndexTable);
@ -189,7 +187,7 @@ class WordDictionary extends AbstractDictionary {
int[] buffer = new int[3];
byte[] intBuffer = new byte[4];
String tmpword;
RandomAccessFile dctFile = new RandomAccessFile(dctFilePath, "r");
DataInputStream dctFile = new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)));
// GB2312 characters 0 - 6768
for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {

14
lucene/analysis/stempel/src/java/org/egothor/stemmer/Compile.java
View File

@ -55,14 +55,11 @@
package org.egothor.stemmer;
import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Locale;
import java.util.StringTokenizer;
@ -126,8 +123,7 @@ public class Compile {
allocTrie();
System.out.println(args[i]);
in = new LineNumberReader(new BufferedReader(new InputStreamReader(
new FileInputStream(args[i]), charset)));
in = new LineNumberReader(Files.newBufferedReader(Paths.get(args[i]), Charset.forName(charset)));
for (String line = in.readLine(); line != null; line = in.readLine()) {
try {
line = line.toLowerCase(Locale.ROOT);
@ -186,7 +182,7 @@ public class Compile {
}
DataOutputStream os = new DataOutputStream(new BufferedOutputStream(
new FileOutputStream(args[i] + ".out")));
Files.newOutputStream(Paths.get(args[i] + ".out"))));
os.writeUTF(args[0]);
trie.store(os);
os.close();

8
lucene/analysis/stempel/src/java/org/egothor/stemmer/DiffIt.java
View File

@ -54,10 +54,10 @@
*/
package org.egothor.stemmer;
import java.io.BufferedReader;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Locale;
import java.util.StringTokenizer;
@ -99,7 +99,7 @@ public class DiffIt {
// System.out.println("[" + args[i] + "]");
Diff diff = new Diff(ins, del, rep, nop);
String charset = System.getProperty("egothor.stemmer.charset", "UTF-8");
in = new LineNumberReader(new BufferedReader(new InputStreamReader(new FileInputStream(args[i]), charset)));
in = new LineNumberReader(Files.newBufferedReader(Paths.get(args[i]), Charset.forName(charset)));
for (String line = in.readLine(); line != null; line = in.readLine()) {
try {
line = line.toLowerCase(Locale.ROOT);

89
lucene/analysis/stempel/src/test/org/egothor/stemmer/TestCompile.java
View File

@ -56,77 +56,66 @@ package org.egothor.stemmer;
*/
import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.DataInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.LineNumberReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Locale;
import java.util.StringTokenizer;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
public class TestCompile extends LuceneTestCase {
public void testCompile() throws Exception {
File dir = createTempDir("testCompile");
dir.mkdirs();
InputStream input = getClass().getResourceAsStream("testRules.txt");
File output = new File(dir, "testRules.txt");
copy(input, output);
input.close();
String path = output.getAbsolutePath();
Path dir = createTempDir("testCompile");
Path output = dir.resolve("testRules.txt");
try (InputStream input = getClass().getResourceAsStream("testRules.txt")) {
Files.copy(input, output);
}
String path = output.toAbsolutePath().toString();
Compile.main(new String[] {"test", path});
String compiled = path + ".out";
Path compiled = dir.resolve("testRules.txt.out");
Trie trie = loadTrie(compiled);
assertTrie(trie, path, true, true);
assertTrie(trie, path, false, true);
Files.delete(new File(compiled).toPath());
assertTrie(trie, output, true, true);
assertTrie(trie, output, false, true);
}
public void testCompileBackwards() throws Exception {
File dir = createTempDir("testCompile");
dir.mkdirs();
InputStream input = getClass().getResourceAsStream("testRules.txt");
File output = new File(dir, "testRules.txt");
copy(input, output);
input.close();
String path = output.getAbsolutePath();
Path dir = createTempDir("testCompile");
Path output = dir.resolve("testRules.txt");
try (InputStream input = getClass().getResourceAsStream("testRules.txt")) {
Files.copy(input, output);
}
String path = output.toAbsolutePath().toString();
Compile.main(new String[] {"-test", path});
String compiled = path + ".out";
Path compiled = dir.resolve("testRules.txt.out");
Trie trie = loadTrie(compiled);
assertTrie(trie, path, true, true);
assertTrie(trie, path, false, true);
Files.delete(new File(compiled).toPath());
assertTrie(trie, output, true, true);
assertTrie(trie, output, false, true);
}
public void testCompileMulti() throws Exception {
File dir = createTempDir("testCompile");
dir.mkdirs();
InputStream input = getClass().getResourceAsStream("testRules.txt");
File output = new File(dir, "testRules.txt");
copy(input, output);
input.close();
String path = output.getAbsolutePath();
Path dir = createTempDir("testCompile");
Path output = dir.resolve("testRules.txt");
try (InputStream input = getClass().getResourceAsStream("testRules.txt")) {
Files.copy(input, output);
}
String path = output.toAbsolutePath().toString();
Compile.main(new String[] {"Mtest", path});
String compiled = path + ".out";
Path compiled = dir.resolve("testRules.txt.out");
Trie trie = loadTrie(compiled);
assertTrie(trie, path, true, true);
assertTrie(trie, path, false, true);
Files.delete(new File(compiled).toPath());
assertTrie(trie, output, true, true);
assertTrie(trie, output, false, true);
}
static Trie loadTrie(String path) throws IOException {
static Trie loadTrie(Path path) throws IOException {
Trie trie;
DataInputStream is = new DataInputStream(new BufferedInputStream(
new FileInputStream(path)));
Files.newInputStream(path)));
String method = is.readUTF().toUpperCase(Locale.ROOT);
if (method.indexOf('M') < 0) {
trie = new Trie(is);
@ -137,10 +126,9 @@ public class TestCompile extends LuceneTestCase {
return trie;
}
private static void assertTrie(Trie trie, String file, boolean usefull,
private static void assertTrie(Trie trie, Path file, boolean usefull,
boolean storeorig) throws Exception {
LineNumberReader in = new LineNumberReader(new BufferedReader(
new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8)));
LineNumberReader in = new LineNumberReader(Files.newBufferedReader(file, StandardCharsets.UTF_8));
for (String line = in.readLine(); line != null; line = in.readLine()) {
try {
@ -172,17 +160,4 @@ public class TestCompile extends LuceneTestCase {
in.close();
}
private static void copy(InputStream input, File output) throws IOException {
FileOutputStream os = new FileOutputStream(output);
try {
byte buffer[] = new byte[1024];
int len;
while ((len = input.read(buffer)) > 0) {
os.write(buffer, 0, len);
}
} finally {
os.close();
}
}
}

34
lucene/backward-codecs/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
View File

@ -18,14 +18,14 @@ package org.apache.lucene.index;
*/
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.lang.reflect.Modifier;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@ -294,8 +294,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
names.addAll(Arrays.asList(oldSingleSegmentNames));
oldIndexDirs = new HashMap<>();
for (String name : names) {
File dir = createTempDir(name);
File dataFile = new File(TestBackwardsCompatibility.class.getResource("index." + name + ".zip").toURI());
Path dir = createTempDir(name);
Path dataFile = Paths.get(TestBackwardsCompatibility.class.getResource("index." + name + ".zip").toURI());
TestUtil.unzip(dataFile, dir);
oldIndexDirs.put(name, newFSDirectory(dir));
}
@ -434,9 +434,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
if (VERBOSE) {
System.out.println("TEST: index " + unsupportedNames[i]);
}
File oldIndxeDir = createTempDir(unsupportedNames[i]);
TestUtil.unzip(getDataFile("unsupported." + unsupportedNames[i] + ".zip"), oldIndxeDir);
BaseDirectoryWrapper dir = newFSDirectory(oldIndxeDir);
Path oldIndexDir = createTempDir(unsupportedNames[i]);
TestUtil.unzip(getDataPath("unsupported." + unsupportedNames[i] + ".zip"), oldIndexDir);
BaseDirectoryWrapper dir = newFSDirectory(oldIndexDir);
// don't checkindex, these are intentionally not supported
dir.setCheckIndexOnClose(false);
@ -487,7 +487,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
assertTrue(bos.toString(IOUtils.UTF_8).contains(IndexFormatTooOldException.class.getName()));
dir.close();
IOUtils.rm(oldIndxeDir);
IOUtils.rm(oldIndexDir);
}
}
@ -795,9 +795,9 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
reader.close();
}
public File createIndex(String dirName, boolean doCFS, boolean fullyMerged) throws IOException {
public Path createIndex(String dirName, boolean doCFS, boolean fullyMerged) throws IOException {
// we use a real directory name that is not cleaned up, because this method is only used to create backwards indexes:
File indexDir = new File("/tmp/idx", dirName);
Path indexDir = Paths.get("/tmp/idx").resolve(dirName);
IOUtils.rm(indexDir);
Directory dir = newFSDirectory(indexDir);
LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
@ -1085,11 +1085,11 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
System.setOut(new PrintStream(new ByteArrayOutputStream(), false, "UTF-8"));
try {
for (String name : oldIndexDirs.keySet()) {
File dir = createTempDir(name);
File dataFile = new File(TestBackwardsCompatibility.class.getResource("index." + name + ".zip").toURI());
Path dir = createTempDir(name);
Path dataFile = Paths.get(TestBackwardsCompatibility.class.getResource("index." + name + ".zip").toURI());
TestUtil.unzip(dataFile, dir);
String path = dir.getAbsolutePath();
String path = dir.toAbsolutePath().toString();
List<String> args = new ArrayList<>();
if (random().nextBoolean()) {
@ -1196,8 +1196,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
public static final String moreTermsIndex = "moreterms.40.zip";
public void testMoreTerms() throws Exception {
File oldIndexDir = createTempDir("moreterms");
TestUtil.unzip(getDataFile(moreTermsIndex), oldIndexDir);
Path oldIndexDir = createTempDir("moreterms");
TestUtil.unzip(getDataPath(moreTermsIndex), oldIndexDir);
Directory dir = newFSDirectory(oldIndexDir);
// TODO: more tests
TestUtil.checkIndex(dir);
@ -1235,8 +1235,8 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
public void testDocValuesUpdates() throws Exception {
File oldIndexDir = createTempDir("dvupdates");
TestUtil.unzip(getDataFile(dvUpdatesIndex), oldIndexDir);
Path oldIndexDir = createTempDir("dvupdates");
TestUtil.unzip(getDataPath(dvUpdatesIndex), oldIndexDir);
Directory dir = newFSDirectory(oldIndexDir);
verifyDocValues(dir);

16
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/Benchmark.java
View File

@ -17,14 +17,14 @@ package org.apache.lucene.benchmark.byTask;
* limitations under the License.
*/
import java.io.File;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.lucene.benchmark.byTask.utils.Algorithm;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.util.IOUtils;
/**
* Run the benchmark algorithm.
@ -97,17 +97,17 @@ public class Benchmark {
}
// verify input files
File algFile = new File(args[0]);
if (!algFile.exists() || !algFile.isFile() || !algFile.canRead()) {
System.err.println("cannot find/read algorithm file: "+algFile.getAbsolutePath());
Path algFile = Paths.get(args[0]);
if (!Files.isReadable(algFile)) {
System.err.println("cannot find/read algorithm file: "+algFile.toAbsolutePath());
System.exit(1);
}
System.out.println("Running algorithm from: "+algFile.getAbsolutePath());
System.out.println("Running algorithm from: "+algFile.toAbsolutePath());
Benchmark benchmark = null;
try {
benchmark = new Benchmark(IOUtils.getDecodingReader(algFile, StandardCharsets.UTF_8));
benchmark = new Benchmark(Files.newBufferedReader(algFile, StandardCharsets.UTF_8));
} catch (Exception e) {
e.printStackTrace();
System.exit(1);

12
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/PerfRunData.java
View File

@ -18,8 +18,10 @@ package org.apache.lucene.benchmark.byTask;
*/
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Locale;
@ -191,12 +193,12 @@ public class PerfRunData implements Closeable {
private Directory createDirectory(boolean eraseIndex, String dirName,
String dirParam) throws IOException {
if ("FSDirectory".equals(config.get(dirParam,"RAMDirectory"))) {
File workDir = new File(config.get("work.dir","work"));
File indexDir = new File(workDir,dirName);
if (eraseIndex && indexDir.exists()) {
Path workDir = Paths.get(config.get("work.dir","work"));
Path indexDir = workDir.resolve(dirName);
if (eraseIndex && Files.exists(indexDir)) {
IOUtils.rm(indexDir);
}
indexDir.mkdirs();
Files.createDirectories(indexDir);
return FSDirectory.open(indexDir);
}

32
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ContentItemsSource.java
View File

@ -18,10 +18,13 @@ package org.apache.lucene.benchmark.byTask.feeds;
*/
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
import java.nio.file.FileVisitResult;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.SimpleFileVisitor;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.ArrayList;
import java.util.Arrays;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.benchmark.byTask.utils.Format;
@ -78,24 +81,19 @@ public abstract class ContentItemsSource implements Closeable {
/**
* A convenience method for collecting all the files of a content source from
* a given directory. The collected {@link File} instances are stored in the
* a given directory. The collected {@link Path} instances are stored in the
* given <code>files</code>.
*/
protected final void collectFiles(File dir, ArrayList<File> files) {
if (!dir.canRead()) {
return;
}
File[] dirFiles = dir.listFiles();
Arrays.sort(dirFiles);
for (int i = 0; i < dirFiles.length; i++) {
File file = dirFiles[i];
if (file.isDirectory()) {
collectFiles(file, files);
} else if (file.canRead()) {
files.add(file);
protected final void collectFiles(Path dir, final ArrayList<Path> files) throws IOException {
Files.walkFileTree(dir, new SimpleFileVisitor<Path>() {
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
if (Files.isReadable(file)) {
files.add(file.toRealPath());
}
return FileVisitResult.CONTINUE;
}
}
});
}
/**

90
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java
View File

@ -20,17 +20,19 @@ package org.apache.lucene.benchmark.byTask.feeds;
import org.apache.lucene.benchmark.byTask.utils.Config;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Locale;
import java.util.Stack;
@ -54,11 +56,11 @@ public class DirContentSource extends ContentSource {
/**
* Iterator over the files in the directory
*/
public static class Iterator implements java.util.Iterator<File> {
public static class Iterator implements java.util.Iterator<Path> {
static class Comparator implements java.util.Comparator<File> {
static class Comparator implements java.util.Comparator<Path> {
@Override
public int compare(File _a, File _b) {
public int compare(Path _a, Path _b) {
String a = _a.toString();
String b = _b.toString();
int diff = a.length() - b.length();
@ -82,47 +84,49 @@ public class DirContentSource extends ContentSource {
int count = 0;
Stack<File> stack = new Stack<>();
Stack<Path> stack = new Stack<>();
/* this seems silly ... there must be a better way ...
not that this is good, but can it matter? */
Comparator c = new Comparator();
public Iterator(File f) {
public Iterator(Path f) throws IOException {
push(f);
}
void find() {
void find() throws IOException {
if (stack.empty()) {
return;
}
if (!(stack.peek()).isDirectory()) {
if (!Files.isDirectory(stack.peek())) {
return;
}
File f = stack.pop();
Path f = stack.pop();
push(f);
}
void push(File f) {
push(f.listFiles(new FileFilter() {
@Override
public boolean accept(File file) {
return file.isDirectory();
void push(Path f) throws IOException {
try (DirectoryStream<Path> stream = Files.newDirectoryStream(f)) {
List<Path> found = new ArrayList<>();
for (Path p : stream) {
if (Files.isDirectory(p)) {
found.add(p);
}
}
}));
push(f.listFiles(new FileFilter() {
@Override
public boolean accept(File file) {
return file.getName().endsWith(".txt");
push(found.toArray(new Path[found.size()]));
}
try (DirectoryStream<Path> stream = Files.newDirectoryStream(f, "*.txt")) {
List<Path> found = new ArrayList<>();
for (Path p : stream) {
found.add(p);
}
}));
push(found.toArray(new Path[found.size()]));
}
find();
}
void push(File[] files) {
void push(Path[] files) {
Arrays.sort(files, c);
for(int i = 0; i < files.length; i++) {
// System.err.println("push " + files[i]);
@ -140,12 +144,16 @@ public class DirContentSource extends ContentSource {
}
@Override
public File next() {
public Path next() {
assert hasNext();
count++;
File object = stack.pop();
Path object = stack.pop();
// System.err.println("pop " + object);
find();
try {
find();
} catch (IOException e) {
throw new RuntimeException(e);
}
return object;
}
@ -157,7 +165,7 @@ public class DirContentSource extends ContentSource {
}
private ThreadLocal<DateFormatInfo> dateFormat = new ThreadLocal<>();
private File dataDir = null;
private Path dataDir = null;
private int iteration = 0;
private Iterator inputFiles = null;
@ -190,7 +198,7 @@ public class DirContentSource extends ContentSource {
@Override
public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException {
File f = null;
Path f = null;
String name = null;
synchronized (this) {
if (!inputFiles.hasNext()) {
@ -203,10 +211,10 @@ public class DirContentSource extends ContentSource {
}
f = inputFiles.next();
// System.err.println(f);
name = f.getCanonicalPath()+"_"+iteration;
name = f.toRealPath()+"_"+iteration;
}
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(f), StandardCharsets.UTF_8));
BufferedReader reader = Files.newBufferedReader(f, StandardCharsets.UTF_8);
String line = null;
//First line is the date, 3rd is the title, rest is body
String dateStr = reader.readLine();
@ -218,7 +226,7 @@ public class DirContentSource extends ContentSource {
bodyBuf.append(line).append(' ');
}
reader.close();
addBytes(f.length());
addBytes(Files.size(f));
Date date = parseDate(dateStr);
@ -241,17 +249,21 @@ public class DirContentSource extends ContentSource {
public void setConfig(Config config) {
super.setConfig(config);
File workDir = new File(config.get("work.dir", "work"));
Path workDir = Paths.get(config.get("work.dir", "work"));
String d = config.get("docs.dir", "dir-out");
dataDir = new File(d);
dataDir = Paths.get(d);
if (!dataDir.isAbsolute()) {
dataDir = new File(workDir, d);
dataDir = workDir.resolve(d);
}
inputFiles = new Iterator(dataDir);
try {
inputFiles = new Iterator(dataDir);
} catch (IOException e) {
throw new RuntimeException(e);
}
if (inputFiles == null) {
throw new RuntimeException("No txt files in dataDir: " + dataDir.getAbsolutePath());
throw new RuntimeException("No txt files in dataDir: " + dataDir.toAbsolutePath());
}
}

7
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/EnwikiContentSource.java
View File

@ -17,10 +17,11 @@ package org.apache.lucene.benchmark.byTask.feeds;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@ -280,7 +281,7 @@ public class EnwikiContentSource extends ContentSource {
return val == null ? -1 : val.intValue();
}
private File file;
private Path file;
private boolean keepImages = true;
private InputStream is;
private Parser parser = new Parser();
@ -324,7 +325,7 @@ public class EnwikiContentSource extends ContentSource {
keepImages = config.get("keep.image.only.docs", true);
String fileName = config.get("docs.file", null);
if (fileName != null) {
file = new File(fileName).getAbsoluteFile();
file = Paths.get(fileName).toAbsolutePath();
}
}

9
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/FileBasedQueryMaker.java
View File

@ -9,6 +9,9 @@ import org.apache.lucene.util.IOUtils;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
@ -58,11 +61,11 @@ public class FileBasedQueryMaker extends AbstractQueryMaker implements QueryMake
String fileName = config.get("file.query.maker.file", null);
if (fileName != null)
{
File file = new File(fileName);
Path path = Paths.get(fileName);
Reader reader = null;
// note: we use a decoding reader, so if your queries are screwed up you know
if (file.exists()) {
reader = IOUtils.getDecodingReader(file, StandardCharsets.UTF_8);
if (Files.exists(path)) {
reader = Files.newBufferedReader(path, StandardCharsets.UTF_8);
} else {
//see if we can find it as a resource
InputStream asStream = FileBasedQueryMaker.class.getClassLoader().getResourceAsStream(fileName);

7
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/LineDocSource.java
View File

@ -18,11 +18,12 @@ package org.apache.lucene.benchmark.byTask.feeds;
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Properties;
@ -170,7 +171,7 @@ public class LineDocSource extends ContentSource {
}
}
private File file;
private Path file;
private BufferedReader reader;
private int readCount;
@ -276,7 +277,7 @@ public class LineDocSource extends ContentSource {
if (fileName == null) {
throw new IllegalArgumentException("docs.file must be set");
}
file = new File(fileName).getAbsoluteFile();
file = Paths.get(fileName).toAbsolutePath();
if (encoding == null) {
encoding = IOUtils.UTF_8;
}

35
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/ReutersContentSource.java
View File

@ -18,11 +18,11 @@ package org.apache.lucene.benchmark.byTask.feeds;
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
@ -50,24 +50,28 @@ public class ReutersContentSource extends ContentSource {
}
private ThreadLocal<DateFormatInfo> dateFormat = new ThreadLocal<>();
private File dataDir = null;
private ArrayList<File> inputFiles = new ArrayList<>();
private Path dataDir = null;
private ArrayList<Path> inputFiles = new ArrayList<>();
private int nextFile = 0;
private int iteration = 0;
@Override
public void setConfig(Config config) {
super.setConfig(config);
File workDir = new File(config.get("work.dir", "work"));
Path workDir = Paths.get(config.get("work.dir", "work"));
String d = config.get("docs.dir", "reuters-out");
dataDir = new File(d);
dataDir = Paths.get(d);
if (!dataDir.isAbsolute()) {
dataDir = new File(workDir, d);
dataDir = workDir.resolve(d);
}
inputFiles.clear();
collectFiles(dataDir, inputFiles);
try {
collectFiles(dataDir, inputFiles);
} catch (IOException e) {
throw new RuntimeException(e);
}
if (inputFiles.size() == 0) {
throw new RuntimeException("No txt files in dataDir: "+dataDir.getAbsolutePath());
throw new RuntimeException("No txt files in dataDir: "+dataDir.toAbsolutePath());
}
}
@ -99,7 +103,7 @@ public class ReutersContentSource extends ContentSource {
@Override
public DocData getNextDocData(DocData docData) throws NoMoreDataException, IOException {
File f = null;
Path f = null;
String name = null;
synchronized (this) {
if (nextFile >= inputFiles.size()) {
@ -111,11 +115,10 @@ public class ReutersContentSource extends ContentSource {
iteration++;
}
f = inputFiles.get(nextFile++);
name = f.getCanonicalPath() + "_" + iteration;
name = f.toRealPath() + "_" + iteration;
}
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(f), StandardCharsets.UTF_8));
try {
try (BufferedReader reader = Files.newBufferedReader(f, StandardCharsets.UTF_8)) {
// First line is the date, 3rd is the title, rest is body
String dateStr = reader.readLine();
reader.readLine();// skip an empty line
@ -128,7 +131,7 @@ public class ReutersContentSource extends ContentSource {
}
reader.close();
addBytes(f.length());
addBytes(Files.size(f));
Date date = parseDate(dateStr.trim());
@ -138,8 +141,6 @@ public class ReutersContentSource extends ContentSource {
docData.setTitle(title);
docData.setDate(date);
return docData;
} finally {
reader.close();
}
}

26
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecContentSource.java
View File

@ -18,11 +18,13 @@ package org.apache.lucene.benchmark.byTask.feeds;
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.DateFormat;
import java.text.ParsePosition;
import java.text.SimpleDateFormat;
@ -82,8 +84,8 @@ public class TrecContentSource extends ContentSource {
private ThreadLocal<DateFormatInfo> dateFormats = new ThreadLocal<>();
private ThreadLocal<StringBuilder> trecDocBuffer = new ThreadLocal<>();
private File dataDir = null;
private ArrayList<File> inputFiles = new ArrayList<>();
private Path dataDir = null;
private ArrayList<Path> inputFiles = new ArrayList<>();
private int nextFile = 0;
// Use to synchronize threads on reading from the TREC documents.
private Object lock = new Object();
@ -174,9 +176,9 @@ public class TrecContentSource extends ContentSource {
nextFile = 0;
iteration++;
}
File f = inputFiles.get(nextFile++);
Path f = inputFiles.get(nextFile++);
if (verbose) {
System.out.println("opening: " + f + " length: " + f.length());
System.out.println("opening: " + f + " length: " + Files.size(f));
}
try {
InputStream inputStream = StreamUtils.inputStream(f); // support either gzip, bzip2, or regular text file, by extension
@ -185,7 +187,7 @@ public class TrecContentSource extends ContentSource {
return;
} catch (Exception e) {
if (verbose) {
System.out.println("Skipping 'bad' file " + f.getAbsolutePath()+" due to "+e.getMessage());
System.out.println("Skipping 'bad' file " + f.toAbsolutePath()+" due to "+e.getMessage());
continue;
}
throw new NoMoreDataException();
@ -291,14 +293,18 @@ public class TrecContentSource extends ContentSource {
public void setConfig(Config config) {
super.setConfig(config);
// dirs
File workDir = new File(config.get("work.dir", "work"));
Path workDir = Paths.get(config.get("work.dir", "work"));
String d = config.get("docs.dir", "trec");
dataDir = new File(d);
dataDir = Paths.get(d);
if (!dataDir.isAbsolute()) {
dataDir = new File(workDir, d);
dataDir = workDir.resolve(d);
}
// files
collectFiles(dataDir, inputFiles);
try {
collectFiles(dataDir, inputFiles);
} catch (IOException e) {
throw new RuntimeException(e);
}
if (inputFiles.size() == 0) {
throw new IllegalArgumentException("No files in dataDir: " + dataDir);
}

8
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/TrecDocParser.java
View File

@ -17,8 +17,8 @@ package org.apache.lucene.benchmark.byTask.feeds;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@ -57,14 +57,14 @@ public abstract class TrecDocParser {
/**
* Compute the path type of a file by inspecting name of file and its parents
*/
public static ParsePathType pathType(File f) {
public static ParsePathType pathType(Path f) {
int pathLength = 0;
while (f != null && ++pathLength < MAX_PATH_LENGTH) {
ParsePathType ppt = pathName2Type.get(f.getName().toUpperCase(Locale.ROOT));
ParsePathType ppt = pathName2Type.get(f.getFileName().toString().toUpperCase(Locale.ROOT));
if (ppt!=null) {
return ppt;
}
f = f.getParentFile();
f = f.getParent();
}
return DEFAULT_PATH_TYPE;
}

4
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTask.java
View File

@ -17,7 +17,7 @@ package org.apache.lucene.benchmark.byTask.tasks;
* limitations under the License.
*/
import java.io.File;
import java.nio.file.Paths;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.index.DirectoryReader;
@ -54,7 +54,7 @@ public class AddIndexesTask extends PerfTask {
if (inputDirProp == null) {
throw new IllegalArgumentException("config parameter " + ADDINDEXES_INPUT_DIR + " not specified in configuration");
}
inputDir = FSDirectory.open(new File(inputDirProp));
inputDir = FSDirectory.open(Paths.get(inputDirProp));
}
@Override

10
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/AnalyzerFactoryTask.java
View File

@ -27,9 +27,11 @@ import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.utils.AnalyzerFactory;
import org.apache.lucene.util.Version;
import java.io.File;
import java.io.StreamTokenizer;
import java.io.StringReader;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@ -364,9 +366,9 @@ public class AnalyzerFactoryTask extends PerfTask {
throw new RuntimeException("Line #" + lineno(stok) + ": ", e);
}
if (instance instanceof ResourceLoaderAware) {
File baseDir = new File(getRunData().getConfig().get("work.dir", "work")).getAbsoluteFile();
if ( ! baseDir.isDirectory()) {
baseDir = new File(".").getAbsoluteFile();
Path baseDir = Paths.get(getRunData().getConfig().get("work.dir", "work"));
if (!Files.isDirectory(baseDir)) {
baseDir = Paths.get(".");
}
((ResourceLoaderAware)instance).inform(new FilesystemResourceLoader(baseDir));
}

9
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
View File

@ -37,11 +37,12 @@ import org.apache.lucene.index.NoMergeScheduler;
import org.apache.lucene.util.Version;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
/**
* Create an index. <br>
@ -191,8 +192,8 @@ public class CreateIndexTask extends PerfTask {
} else if (infoStreamVal.equals("SystemErr")) {
iwc.setInfoStream(System.err);
} else {
File f = new File(infoStreamVal).getAbsoluteFile();
iwc.setInfoStream(new PrintStream(new BufferedOutputStream(new FileOutputStream(f)), false, Charset.defaultCharset().name()));
Path f = Paths.get(infoStreamVal);
iwc.setInfoStream(new PrintStream(new BufferedOutputStream(Files.newOutputStream(f)), false, Charset.defaultCharset().name()));
}
}
IndexWriter writer = new IndexWriter(runData.getDirectory(), iwc);

12
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTask.java
View File

@ -6,6 +6,8 @@ import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Path;
import java.nio.file.Paths;
import org.apache.lucene.benchmark.byTask.PerfRunData;
import org.apache.lucene.benchmark.byTask.feeds.DocMaker;
@ -41,16 +43,16 @@ public class WriteEnwikiLineDocTask extends WriteLineDocTask {
public WriteEnwikiLineDocTask(PerfRunData runData) throws Exception {
super(runData);
OutputStream out = StreamUtils.outputStream(categoriesLineFile(new File(fname)));
OutputStream out = StreamUtils.outputStream(categoriesLineFile(Paths.get(fname)));
categoryLineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8), StreamUtils.BUFFER_SIZE));
writeHeader(categoryLineFileOut);
}
/** Compose categories line file out of original line file */
public static File categoriesLineFile(File f) {
File dir = f.getParentFile();
String categoriesName = "categories-"+f.getName();
return dir==null ? new File(categoriesName) : new File(dir,categoriesName);
public static Path categoriesLineFile(Path f) {
Path dir = f.toAbsolutePath().getParent();
String categoriesName = "categories-"+f.getFileName();
return dir.resolve(categoriesName);
}
@Override

4
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTask.java
View File

@ -18,11 +18,11 @@ package org.apache.lucene.benchmark.byTask.tasks;
*/
import java.io.BufferedWriter;
import java.io.File;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.HashSet;
import java.util.regex.Matcher;
@ -101,7 +101,7 @@ public class WriteLineDocTask extends PerfTask {
if (fname == null) {
throw new IllegalArgumentException("line.file.out must be set");
}
OutputStream out = StreamUtils.outputStream(new File(fname));
OutputStream out = StreamUtils.outputStream(Paths.get(fname));
lineFileOut = new PrintWriter(new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8), StreamUtils.BUFFER_SIZE));
docMaker = runData.getDocMaker();

19
lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/utils/StreamUtils.java
View File

@ -19,12 +19,11 @@ package org.apache.lucene.benchmark.byTask.utils;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Locale;
import java.util.Map;
@ -83,17 +82,17 @@ public class StreamUtils {
* based on the file name (e.g., if it ends with .bz2 or .bzip, return a
* 'bzip' {@link InputStream}).
*/
public static InputStream inputStream(File file) throws IOException {
public static InputStream inputStream(Path file) throws IOException {
// First, create a FileInputStream, as this will be required by all types.
// Wrap with BufferedInputStream for better performance
InputStream in = new BufferedInputStream(new FileInputStream(file), BUFFER_SIZE);
InputStream in = new BufferedInputStream(Files.newInputStream(file), BUFFER_SIZE);
return fileType(file).inputStream(in);
}
/** Return the type of the file, or null if unknown */
private static Type fileType(File file) {
private static Type fileType(Path file) {
Type type = null;
String fileName = file.getName();
String fileName = file.getFileName().toString();
int idx = fileName.lastIndexOf('.');
if (idx != -1) {
type = extensionToType.get(fileName.substring(idx).toLowerCase(Locale.ROOT));
@ -103,12 +102,12 @@ public class StreamUtils {
/**
* Returns an {@link OutputStream} over the requested file, identifying
* the appropriate {@link OutputStream} instance similar to {@link #inputStream(File)}.
* the appropriate {@link OutputStream} instance similar to {@link #inputStream(Path)}.
*/
public static OutputStream outputStream(File file) throws IOException {
public static OutputStream outputStream(Path file) throws IOException {
// First, create a FileInputStream, as this will be required by all types.
// Wrap with BufferedInputStream for better performance
OutputStream os = new BufferedOutputStream(new FileOutputStream(file), BUFFER_SIZE);
OutputStream os = new BufferedOutputStream(Files.newOutputStream(file), BUFFER_SIZE);
return fileType(file).outputStream(os);
}
}

15
lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
View File

@ -26,12 +26,13 @@ import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.IOUtils;
import java.io.BufferedReader;
import java.io.File;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.nio.charset.Charset;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.HashSet;
import java.util.Set;
@ -52,10 +53,10 @@ public class QueryDriver {
System.exit(1);
}
File topicsFile = new File(args[0]);
File qrelsFile = new File(args[1]);
Path topicsFile = Paths.get(args[0]);
Path qrelsFile = Paths.get(args[1]);
SubmissionReport submitLog = new SubmissionReport(new PrintWriter(args[2], IOUtils.UTF_8 /* huh, no nio.Charset ctor? */), "lucene");
FSDirectory dir = FSDirectory.open(new File(args[3]));
FSDirectory dir = FSDirectory.open(Paths.get(args[3]));
String fieldSpec = args.length == 5 ? args[4] : "T"; // default to Title-only if not specified.
IndexReader reader = DirectoryReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);
@ -67,10 +68,10 @@ public class QueryDriver {
// use trec utilities to read trec topics into quality queries
TrecTopicsReader qReader = new TrecTopicsReader();
QualityQuery qqs[] = qReader.readQueries(new BufferedReader(IOUtils.getDecodingReader(topicsFile, StandardCharsets.UTF_8)));
QualityQuery qqs[] = qReader.readQueries(Files.newBufferedReader(topicsFile, StandardCharsets.UTF_8));
// prepare judge, with trec utilities that read from a QRels file
Judge judge = new TrecJudge(new BufferedReader(IOUtils.getDecodingReader(qrelsFile, StandardCharsets.UTF_8)));
Judge judge = new TrecJudge(Files.newBufferedReader(qrelsFile, StandardCharsets.UTF_8));
// validate topics & judgments match each other
judge.validateData(qqs, logger);

4
lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/QualityQueriesFinder.java
View File

@ -16,8 +16,8 @@
*/
package org.apache.lucene.benchmark.quality.utils;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@ -54,7 +54,7 @@ public class QualityQueriesFinder {
System.err.println("Usage: java QualityQueriesFinder <index-dir>");
System.exit(1);
}
QualityQueriesFinder qqf = new QualityQueriesFinder(FSDirectory.open(new File(args[0])));
QualityQueriesFinder qqf = new QualityQueriesFinder(FSDirectory.open(Paths.get(args[0])));
String q[] = qqf.bestQueries("body",20);
for (int i=0; i<q.length; i++) {
System.out.println(newline+formatQueryAsTrecTopic(i,q[i],null,null));

70
lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractReuters.java
View File

@ -17,48 +17,43 @@ package org.apache.lucene.benchmark.utils;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.BufferedWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import org.apache.lucene.util.IOUtils;
/**
* Split the Reuters SGML documents into Simple Text files containing: Title, Date, Dateline, Body
*/
public class ExtractReuters {
private File reutersDir;
private File outputDir;
private static final String LINE_SEPARATOR = System.getProperty("line.separator");
private Path reutersDir;
private Path outputDir;
public ExtractReuters(File reutersDir, File outputDir) throws IOException {
public ExtractReuters(Path reutersDir, Path outputDir) throws IOException {
this.reutersDir = reutersDir;
this.outputDir = outputDir;
System.out.println("Deleting all files in " + outputDir);
for (File f : outputDir.listFiles()) {
Files.delete(f.toPath());
}
IOUtils.rm(outputDir);
}
public void extract() {
File[] sgmFiles = reutersDir.listFiles(new FileFilter() {
@Override
public boolean accept(File file) {
return file.getName().endsWith(".sgm");
}
});
if (sgmFiles != null && sgmFiles.length > 0) {
for (File sgmFile : sgmFiles) {
public void extract() throws IOException {
long count = 0;
try (DirectoryStream<Path> stream = Files.newDirectoryStream(reutersDir, "*.sgm")) {
for (Path sgmFile : stream) {
extractFile(sgmFile);
count++;
}
} else {
}
if (count == 0) {
System.err.println("No .sgm files in " + reutersDir);
}
}
@ -74,10 +69,8 @@ public class ExtractReuters {
/**
* Override if you wish to change what is extracted
*/
protected void extractFile(File sgmFile) {
try {
BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(sgmFile), StandardCharsets.UTF_8));
protected void extractFile(Path sgmFile) {
try (BufferedReader reader = Files.newBufferedReader(sgmFile, StandardCharsets.UTF_8)) {
StringBuilder buffer = new StringBuilder(1024);
StringBuilder outBuffer = new StringBuilder(1024);
@ -101,23 +94,21 @@ public class ExtractReuters {
outBuffer.append(matcher.group(i));
}
}
outBuffer.append(LINE_SEPARATOR).append(LINE_SEPARATOR);
outBuffer.append(System.lineSeparator()).append(System.lineSeparator());
}
String out = outBuffer.toString();
for (int i = 0; i < META_CHARS_SERIALIZATIONS.length; i++) {
out = out.replaceAll(META_CHARS_SERIALIZATIONS[i], META_CHARS[i]);
}
File outFile = new File(outputDir, sgmFile.getName() + "-"
+ (docNumber++) + ".txt");
Path outFile = outputDir.resolve(sgmFile.getFileName() + "-" + (docNumber++) + ".txt");
// System.out.println("Writing " + outFile);
OutputStreamWriter writer = new OutputStreamWriter(new FileOutputStream(outFile), StandardCharsets.UTF_8);
writer.write(out);
writer.close();
try (BufferedWriter writer = Files.newBufferedWriter(outFile, StandardCharsets.UTF_8)) {
writer.write(out);
}
outBuffer.setLength(0);
buffer.setLength(0);
}
}
reader.close();
} catch (IOException e) {
throw new RuntimeException(e);
}
@ -128,21 +119,20 @@ public class ExtractReuters {
usage("Wrong number of arguments ("+args.length+")");
return;
}
File reutersDir = new File(args[0]);
if (!reutersDir.exists()) {
Path reutersDir = Paths.get(args[0]);
if (!Files.exists(reutersDir)) {
usage("Cannot find Path to Reuters SGM files ("+reutersDir+")");
return;
}
// First, extract to a tmp directory and only if everything succeeds, rename
// to output directory.
File outputDir = new File(args[1]);
outputDir = new File(outputDir.getAbsolutePath() + "-tmp");
outputDir.mkdirs();
Path outputDir = Paths.get(args[1] + "-tmp");
Files.createDirectories(outputDir);
ExtractReuters extractor = new ExtractReuters(reutersDir, outputDir);
extractor.extract();
// Now rename to requested output dir
outputDir.renameTo(new File(args[1]));
Files.move(outputDir, Paths.get(args[1]), StandardCopyOption.ATOMIC_MOVE);
}
private static void usage(String msg) {

50
lucene/benchmark/src/java/org/apache/lucene/benchmark/utils/ExtractWikipedia.java
View File

@ -17,13 +17,12 @@ package org.apache.lucene.benchmark.utils;
* limitations under the License.
*/
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Properties;
import org.apache.lucene.benchmark.byTask.feeds.ContentSource;
@ -32,30 +31,28 @@ import org.apache.lucene.benchmark.byTask.feeds.EnwikiContentSource;
import org.apache.lucene.benchmark.byTask.feeds.NoMoreDataException;
import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.document.Document;
import org.apache.lucene.util.IOUtils;
/**
* Extract the downloaded Wikipedia dump into separate files for indexing.
*/
public class ExtractWikipedia {
private File outputDir;
private Path outputDir;
static public int count = 0;
static final int BASE = 10;
protected DocMaker docMaker;
public ExtractWikipedia(DocMaker docMaker, File outputDir) throws IOException {
public ExtractWikipedia(DocMaker docMaker, Path outputDir) throws IOException {
this.outputDir = outputDir;
this.docMaker = docMaker;
System.out.println("Deleting all files in " + outputDir);
File[] files = outputDir.listFiles();
for (int i = 0; i < files.length; i++) {
Files.delete(files[i].toPath());
}
IOUtils.rm(outputDir);
}
public File directory(int count, File directory) {
public Path directory(int count, Path directory) {
if (directory == null) {
directory = outputDir;
}
@ -66,16 +63,16 @@ public class ExtractWikipedia {
if (count < BASE) {
return directory;
}
directory = new File(directory, (Integer.toString(base / BASE)));
directory = new File(directory, (Integer.toString(count / (base / BASE))));
directory = directory.resolve(Integer.toString(base / BASE));
directory = directory.resolve(Integer.toString(count / (base / BASE)));
return directory(count % (base / BASE), directory);
}
public void create(String id, String title, String time, String body) {
public void create(String id, String title, String time, String body) throws IOException {
File d = directory(count++, null);
d.mkdirs();
File f = new File(d, id + ".txt");
Path d = directory(count++, null);
Files.createDirectories(d);
Path f = d.resolve(id + ".txt");
StringBuilder contents = new StringBuilder();
@ -86,14 +83,9 @@ public class ExtractWikipedia {
contents.append(body);
contents.append("\n");
try {
Writer writer = new OutputStreamWriter(new FileOutputStream(f), StandardCharsets.UTF_8);
try (Writer writer = Files.newBufferedWriter(f, StandardCharsets.UTF_8)) {
writer.write(contents.toString());
writer.close();
} catch (IOException ioe) {
throw new RuntimeException(ioe);
}
}
public void extract() throws Exception {
@ -114,16 +106,16 @@ public class ExtractWikipedia {
public static void main(String[] args) throws Exception {
File wikipedia = null;
File outputDir = new File("./enwiki");
Path wikipedia = null;
Path outputDir = Paths.get("enwiki");
boolean keepImageOnlyDocs = true;
for (int i = 0; i < args.length; i++) {
String arg = args[i];
if (arg.equals("--input") || arg.equals("-i")) {
wikipedia = new File(args[i + 1]);
wikipedia = Paths.get(args[i + 1]);
i++;
} else if (arg.equals("--output") || arg.equals("-o")) {
outputDir = new File(args[i + 1]);
outputDir = Paths.get(args[i + 1]);
i++;
} else if (arg.equals("--discardImageOnlyDocs") || arg.equals("-d")) {
keepImageOnlyDocs = false;
@ -131,7 +123,7 @@ public class ExtractWikipedia {
}
Properties properties = new Properties();
properties.setProperty("docs.file", wikipedia.getAbsolutePath());
properties.setProperty("docs.file", wikipedia.toAbsolutePath().toString());
properties.setProperty("content.source.forever", "false");
properties.setProperty("keep.image.only.docs", String.valueOf(keepImageOnlyDocs));
Config config = new Config(properties);
@ -142,9 +134,9 @@ public class ExtractWikipedia {
DocMaker docMaker = new DocMaker();
docMaker.setConfig(config, source);
docMaker.resetInputs();
if (wikipedia.exists()) {
if (Files.exists(wikipedia)) {
System.out.println("Extracting Wikipedia to: " + outputDir + " using EnwikiContentSource");
outputDir.mkdirs();
Files.createDirectories(outputDir);
ExtractWikipedia extractor = new ExtractWikipedia(docMaker, outputDir);
extractor.extract();
} else {

27
lucene/benchmark/src/test/org/apache/lucene/benchmark/BenchmarkTestCase.java
View File

@ -17,12 +17,11 @@ package org.apache.lucene.benchmark;
* limitations under the License.
*/
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.StringReader;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.lucene.benchmark.byTask.Benchmark;
import org.apache.lucene.util.LuceneTestCase;
@ -33,7 +32,7 @@ import org.junit.BeforeClass;
/** Base class for all Benchmark unit tests. */
@SuppressSysoutChecks(bugUrl = "very noisy")
public abstract class BenchmarkTestCase extends LuceneTestCase {
private static File WORKDIR;
private static Path WORKDIR;
@BeforeClass
public static void beforeClassBenchmarkTestCase() {
@ -45,33 +44,27 @@ public abstract class BenchmarkTestCase extends LuceneTestCase {
WORKDIR = null;
}
public File getWorkDir() {
public Path getWorkDir() {
return WORKDIR;
}
/** Copy a resource into the workdir */
public void copyToWorkDir(String resourceName) throws IOException {
InputStream resource = getClass().getResourceAsStream(resourceName);
OutputStream dest = new FileOutputStream(new File(getWorkDir(), resourceName));
byte[] buffer = new byte[8192];
int len;
while ((len = resource.read(buffer)) > 0) {
dest.write(buffer, 0, len);
Path target = getWorkDir().resolve(resourceName);
Files.deleteIfExists(target);
try (InputStream resource = getClass().getResourceAsStream(resourceName)) {
Files.copy(resource, target);
}
resource.close();
dest.close();
}
/** Return a path, suitable for a .alg config file, for a resource in the workdir */
public String getWorkDirResourcePath(String resourceName) {
return new File(getWorkDir(), resourceName).getAbsolutePath().replace("\\", "/");
return getWorkDir().resolve(resourceName).toAbsolutePath().toString().replace("\\", "/");
}
/** Return a path, suitable for a .alg config file, for the workdir */
public String getWorkDirPath() {
return getWorkDir().getAbsolutePath().replace("\\", "/");
return getWorkDir().toAbsolutePath().toString().replace("\\", "/");
}
// create the benchmark and execute it.

18
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
View File

@ -18,11 +18,9 @@
package org.apache.lucene.benchmark.byTask;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.text.Collator;
import java.util.List;
import java.util.Locale;
@ -384,7 +382,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
* Test WriteLineDoc and LineDocSource.
*/
public void testLineDocFile() throws Exception {
File lineFile = createTempFile("test.reuters.lines", ".txt");
Path lineFile = createTempFile("test.reuters.lines", ".txt");
// We will call WriteLineDocs this many times
final int NUM_TRY_DOCS = 50;
@ -394,7 +392,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
"# ----- properties ",
"content.source=org.apache.lucene.benchmark.byTask.feeds.SingleDocSource",
"content.source.forever=true",
"line.file.out=" + lineFile.getAbsolutePath().replace('\\', '/'),
"line.file.out=" + lineFile.toAbsolutePath().toString().replace('\\', '/'),
"# ----- alg ",
"{WriteLineDoc()}:" + NUM_TRY_DOCS,
};
@ -402,9 +400,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
// Run algo
Benchmark benchmark = execBenchmark(algLines1);
BufferedReader r = new BufferedReader(
new InputStreamReader(
new FileInputStream(lineFile), StandardCharsets.UTF_8));
BufferedReader r = Files.newBufferedReader(lineFile, StandardCharsets.UTF_8);
int numLines = 0;
String line;
while((line = r.readLine()) != null) {
@ -421,7 +417,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
"# ----- properties ",
"analyzer=org.apache.lucene.analysis.core.WhitespaceAnalyzer",
"content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
"docs.file=" + lineFile.getAbsolutePath().replace('\\', '/'),
"docs.file=" + lineFile.toAbsolutePath().toString().replace('\\', '/'),
"content.source.forever=false",
"doc.reuse.fields=false",
"ram.flush.mb=4",
@ -445,7 +441,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
assertEquals(numLines + " lines were created but " + ir.numDocs() + " docs are in the index", numLines, ir.numDocs());
ir.close();
Files.delete(lineFile.toPath());
Files.delete(lineFile);
}
/**
@ -1064,7 +1060,7 @@ public class TestPerfTasksLogic extends BenchmarkTestCase {
String algLines[] = {
"content.source=org.apache.lucene.benchmark.byTask.feeds.LineDocSource",
"docs.file=" + getReuters20LinesFile(),
"work.dir=" + getWorkDir().getAbsolutePath().replaceAll("\\\\", "/"), // Fix Windows path
"work.dir=" + getWorkDir().toAbsolutePath().toString().replaceAll("\\\\", "/"), // Fix Windows path
"content.source.forever=false",
"directory=RAMDirectory",
"AnalyzerFactory(name:'" + singleQuoteEscapedName + "', " + params + ")",

28
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksParse.java
View File

@ -17,13 +17,13 @@
package org.apache.lucene.benchmark.byTask;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.StringReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
import org.apache.lucene.benchmark.byTask.feeds.AbstractQueryMaker;
@ -114,22 +114,19 @@ public class TestPerfTasksParse extends LuceneTestCase {
public void testParseExamples() throws Exception {
// hackedy-hack-hack
boolean foundFiles = false;
final File examplesDir = new File(ConfLoader.class.getResource(".").toURI());
for (File algFile : examplesDir.listFiles(new FileFilter() {
@Override
public boolean accept(File pathname) { return pathname.isFile() && pathname.getName().endsWith(".alg"); }
})) {
try {
Config config = new Config(new InputStreamReader(new FileInputStream(algFile), StandardCharsets.UTF_8));
final Path examplesDir = Paths.get(ConfLoader.class.getResource(".").toURI());
try (DirectoryStream<Path> stream = Files.newDirectoryStream(examplesDir, "*.alg")) {
for (Path path : stream) {
Config config = new Config(Files.newBufferedReader(path, StandardCharsets.UTF_8));
String contentSource = config.get("content.source", null);
if (contentSource != null) { Class.forName(contentSource); }
config.set("work.dir", createTempDir(LuceneTestCase.getTestClass().getSimpleName()).getAbsolutePath());
config.set("work.dir", createTempDir(LuceneTestCase.getTestClass().getSimpleName()).toAbsolutePath().toString());
config.set("content.source", MockContentSource.class.getName());
String dir = config.get("content.source", null);
if (dir != null) { Class.forName(dir); }
config.set("directory", RAMDirectory.class.getName());
if (config.get("line.file.out", null) != null) {
config.set("line.file.out", createTempFile("linefile", ".txt").getAbsolutePath());
config.set("line.file.out", createTempFile("linefile", ".txt").toAbsolutePath().toString());
}
if (config.get("query.maker", null) != null) {
Class.forName(config.get("query.maker", null));
@ -137,14 +134,11 @@ public class TestPerfTasksParse extends LuceneTestCase {
}
PerfRunData data = new PerfRunData(config);
new Algorithm(data);
} catch (Throwable t) {
throw new AssertionError("Could not parse sample file: " + algFile, t);
foundFiles = true;
}
foundFiles = true;
}
if (!foundFiles) {
fail("could not find any .alg files!");
}
}
}

9
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/DocMakerTest.java
View File

@ -17,8 +17,9 @@ package org.apache.lucene.benchmark.byTask.feeds;
* limitations under the License.
*/
import java.io.File;
import java.io.PrintStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
@ -166,13 +167,13 @@ public class DocMakerTest extends BenchmarkTestCase {
public void testDocMakerLeak() throws Exception {
// DocMaker did not close its ContentSource if resetInputs was called twice,
// leading to a file handle leak.
File f = new File(getWorkDir(), "docMakerLeak.txt");
PrintStream ps = new PrintStream(f, IOUtils.UTF_8);
Path f = getWorkDir().resolve("docMakerLeak.txt");
PrintStream ps = new PrintStream(Files.newOutputStream(f), true, IOUtils.UTF_8);
ps.println("one title\t" + System.currentTimeMillis() + "\tsome content");
ps.close();
Properties props = new Properties();
props.setProperty("docs.file", f.getAbsolutePath());
props.setProperty("docs.file", f.toAbsolutePath().toString());
props.setProperty("content.source.forever", "false");
Config config = new Config(props);

40
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/LineDocSourceTest.java
View File

@ -18,12 +18,12 @@ package org.apache.lucene.benchmark.byTask.feeds;
*/
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
@ -51,8 +51,8 @@ public class LineDocSourceTest extends BenchmarkTestCase {
private static final CompressorStreamFactory csFactory = new CompressorStreamFactory();
private void createBZ2LineFile(File file, boolean addHeader) throws Exception {
OutputStream out = new FileOutputStream(file);
private void createBZ2LineFile(Path file, boolean addHeader) throws Exception {
OutputStream out = Files.newOutputStream(file);
out = csFactory.createCompressorOutputStream("bzip2", out);
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
writeDocsToFile(writer, addHeader, null);
@ -89,15 +89,15 @@ public class LineDocSourceTest extends BenchmarkTestCase {
writer.newLine();
}
private void createRegularLineFile(File file, boolean addHeader) throws Exception {
OutputStream out = new FileOutputStream(file);
private void createRegularLineFile(Path file, boolean addHeader) throws Exception {
OutputStream out = Files.newOutputStream(file);
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
writeDocsToFile(writer, addHeader, null);
writer.close();
}
private void createRegularLineFileWithMoreFields(File file, String...extraFields) throws Exception {
OutputStream out = new FileOutputStream(file);
private void createRegularLineFileWithMoreFields(Path file, String...extraFields) throws Exception {
OutputStream out = Files.newOutputStream(file);
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
Properties p = new Properties();
for (String f : extraFields) {
@ -107,13 +107,13 @@ public class LineDocSourceTest extends BenchmarkTestCase {
writer.close();
}
private void doIndexAndSearchTest(File file, Class<? extends LineParser> lineParserClass, String storedField) throws Exception {
private void doIndexAndSearchTest(Path file, Class<? extends LineParser> lineParserClass, String storedField) throws Exception {
doIndexAndSearchTestWithRepeats(file, lineParserClass, 1, storedField); // no extra repetitions
doIndexAndSearchTestWithRepeats(file, lineParserClass, 2, storedField); // 1 extra repetition
doIndexAndSearchTestWithRepeats(file, lineParserClass, 4, storedField); // 3 extra repetitions
}
private void doIndexAndSearchTestWithRepeats(File file,
private void doIndexAndSearchTestWithRepeats(Path file,
Class<? extends LineParser> lineParserClass, int numAdds, String storedField) throws Exception {
IndexReader reader = null;
@ -123,7 +123,7 @@ public class LineDocSourceTest extends BenchmarkTestCase {
Properties props = new Properties();
// LineDocSource specific settings.
props.setProperty("docs.file", file.getAbsolutePath());
props.setProperty("docs.file", file.toAbsolutePath().toString());
if (lineParserClass != null) {
props.setProperty("line.parser", lineParserClass.getName());
}
@ -169,31 +169,31 @@ public class LineDocSourceTest extends BenchmarkTestCase {
/* Tests LineDocSource with a bzip2 input stream. */
public void testBZip2() throws Exception {
File file = new File(getWorkDir(), "one-line.bz2");
Path file = getWorkDir().resolve("one-line.bz2");
createBZ2LineFile(file,true);
doIndexAndSearchTest(file, null, null);
}
public void testBZip2NoHeaderLine() throws Exception {
File file = new File(getWorkDir(), "one-line.bz2");
Path file = getWorkDir().resolve("one-line.bz2");
createBZ2LineFile(file,false);
doIndexAndSearchTest(file, null, null);
}
public void testRegularFile() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
createRegularLineFile(file,true);
doIndexAndSearchTest(file, null, null);
}
public void testRegularFileSpecialHeader() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
createRegularLineFile(file,true);
doIndexAndSearchTest(file, HeaderLineParser.class, null);
}
public void testRegularFileNoHeaderLine() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
createRegularLineFile(file,false);
doIndexAndSearchTest(file, null, null);
}
@ -209,8 +209,8 @@ public class LineDocSourceTest extends BenchmarkTestCase {
};
for (int i = 0; i < testCases.length; i++) {
File file = new File(getWorkDir(), "one-line");
BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file), StandardCharsets.UTF_8));
Path file = getWorkDir().resolve("one-line");
BufferedWriter writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8);
writer.write(testCases[i]);
writer.newLine();
writer.close();
@ -225,14 +225,14 @@ public class LineDocSourceTest extends BenchmarkTestCase {
/** Doc Name is not part of the default header */
public void testWithDocsName() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
createRegularLineFileWithMoreFields(file, DocMaker.NAME_FIELD);
doIndexAndSearchTest(file, null, DocMaker.NAME_FIELD);
}
/** Use fields names that are not defined in Docmaker and so will go to Properties */
public void testWithProperties() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
String specialField = "mySpecialField";
createRegularLineFileWithMoreFields(file, specialField);
doIndexAndSearchTest(file, null, specialField);

9
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/feeds/TrecContentSourceTest.java
View File

@ -18,9 +18,9 @@ package org.apache.lucene.benchmark.byTask.feeds;
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.nio.file.Path;
import java.text.ParseException;
import java.util.Arrays;
import java.util.Date;
@ -32,7 +32,6 @@ import org.apache.lucene.benchmark.byTask.utils.Config;
import org.apache.lucene.document.DateTools;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.apache.lucene.util.TestUtil;
public class TrecContentSourceTest extends LuceneTestCase {
@ -344,15 +343,15 @@ public class TrecContentSourceTest extends LuceneTestCase {
* supported formats - bzip, gzip, txt.
*/
public void testTrecFeedDirAllTypes() throws Exception {
File dataDir = createTempDir("trecFeedAllTypes");
TestUtil.unzip(getDataFile("trecdocs.zip"), dataDir);
Path dataDir = createTempDir("trecFeedAllTypes");
TestUtil.unzip(getDataPath("trecdocs.zip"), dataDir);
TrecContentSource tcs = new TrecContentSource();
Properties props = new Properties();
props.setProperty("print.props", "false");
props.setProperty("content.source.verbose", "false");
props.setProperty("content.source.excludeIteration", "true");
props.setProperty("doc.maker.forever", "false");
props.setProperty("docs.dir", dataDir.getCanonicalPath().replace('\\','/'));
props.setProperty("docs.dir", dataDir.toRealPath().toString().replace('\\','/'));
props.setProperty("trec.doc.parser", TrecParserByPath.class.getName());
props.setProperty("content.source.forever", "false");
tcs.setConfig(new Config(props));

8
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/AddIndexesTaskTest.java
View File

@ -17,7 +17,7 @@ package org.apache.lucene.benchmark.byTask.tasks;
* limitations under the License.
*/
import java.io.File;
import java.nio.file.Path;
import java.util.Properties;
import org.apache.lucene.benchmark.BenchmarkTestCase;
@ -36,14 +36,14 @@ import org.junit.BeforeClass;
/** Tests the functionality of {@link AddIndexesTask}. */
public class AddIndexesTaskTest extends BenchmarkTestCase {
private static File testDir, inputDir;
private static Path testDir, inputDir;
@BeforeClass
public static void beforeClassAddIndexesTaskTest() throws Exception {
testDir = createTempDir("addIndexesTask");
// create a dummy index under inputDir
inputDir = new File(testDir, "input");
inputDir = testDir.resolve("input");
Directory tmpDir = newFSDirectory(inputDir);
try {
IndexWriter writer = new IndexWriter(tmpDir, new IndexWriterConfig(null));
@ -61,7 +61,7 @@ public class AddIndexesTaskTest extends BenchmarkTestCase {
props.setProperty("writer.version", Version.LATEST.toString());
props.setProperty("print.props", "false"); // don't print anything
props.setProperty("directory", "RAMDirectory");
props.setProperty(AddIndexesTask.ADDINDEXES_INPUT_DIR, inputDir.getAbsolutePath());
props.setProperty(AddIndexesTask.ADDINDEXES_INPUT_DIR, inputDir.toAbsolutePath().toString());
Config config = new Config(props);
return new PerfRunData(config);
}

9
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTaskTest.java
View File

@ -18,9 +18,10 @@ package org.apache.lucene.benchmark.byTask.tasks;
*/
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.PrintStream;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;
import org.apache.lucene.benchmark.BenchmarkTestCase;
@ -79,11 +80,11 @@ public class CreateIndexTaskTest extends BenchmarkTestCase {
public void testInfoStream_File() throws Exception {
File outFile = new File(getWorkDir(), "infoStreamTest");
PerfRunData runData = createPerfRunData(outFile.getAbsolutePath());
Path outFile = getWorkDir().resolve("infoStreamTest");
PerfRunData runData = createPerfRunData(outFile.toAbsolutePath().toString());
new CreateIndexTask(runData).doLogic();
new CloseIndexTask(runData).doLogic();
assertTrue(outFile.length() > 0);
assertTrue(Files.size(outFile) > 0);
}
public void testNoMergePolicy() throws Exception {

20
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteEnwikiLineDocTaskTest.java
View File

@ -23,6 +23,8 @@ import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
@ -56,26 +58,24 @@ public class WriteEnwikiLineDocTaskTest extends BenchmarkTestCase {
}
private PerfRunData createPerfRunData(File file, String docMakerName) throws Exception {
private PerfRunData createPerfRunData(Path file, String docMakerName) throws Exception {
Properties props = new Properties();
props.setProperty("doc.maker", docMakerName);
props.setProperty("line.file.out", file.getAbsolutePath());
props.setProperty("line.file.out", file.toAbsolutePath().toString());
props.setProperty("directory", "RAMDirectory"); // no accidental FS dir.
Config config = new Config(props);
return new PerfRunData(config);
}
private void doReadTest(File file, String expTitle,
private void doReadTest(Path file, String expTitle,
String expDate, String expBody) throws Exception {
doReadTest(2, file, expTitle, expDate, expBody);
File categoriesFile = WriteEnwikiLineDocTask.categoriesLineFile(file);
Path categoriesFile = WriteEnwikiLineDocTask.categoriesLineFile(file);
doReadTest(2, categoriesFile, "Category:"+expTitle, expDate, expBody);
}
private void doReadTest(int n, File file, String expTitle, String expDate, String expBody) throws Exception {
InputStream in = new FileInputStream(file);
BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
try {
private void doReadTest(int n, Path file, String expTitle, String expDate, String expBody) throws Exception {
try (BufferedReader br = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
String line = br.readLine();
WriteLineDocTaskTest.assertHeaderLine(line);
for (int i=0; i<n; i++) {
@ -91,8 +91,6 @@ public class WriteEnwikiLineDocTaskTest extends BenchmarkTestCase {
}
}
assertNull(br.readLine());
} finally {
br.close();
}
}
@ -101,7 +99,7 @@ public class WriteEnwikiLineDocTaskTest extends BenchmarkTestCase {
// WriteLineDocTask replaced only \t characters w/ a space, since that's its
// separator char. However, it didn't replace newline characters, which
// resulted in errors in LineDocSource.
File file = new File(getWorkDir(), "two-lines-each.txt");
Path file = getWorkDir().resolve("two-lines-each.txt");
PerfRunData runData = createPerfRunData(file, WriteLineCategoryDocMaker.class.getName());
WriteLineDocTask wldt = new WriteEnwikiLineDocTask(runData);
for (int i=0; i<4; i++) { // four times so that each file should have 2 lines.

58
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/tasks/WriteLineDocTaskTest.java
View File

@ -18,11 +18,11 @@ package org.apache.lucene.benchmark.byTask.tasks;
*/
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashSet;
import java.util.Properties;
import java.util.Set;
@ -136,12 +136,12 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
private static final CompressorStreamFactory csFactory = new CompressorStreamFactory();
private PerfRunData createPerfRunData(File file,
private PerfRunData createPerfRunData(Path file,
boolean allowEmptyDocs,
String docMakerName) throws Exception {
Properties props = new Properties();
props.setProperty("doc.maker", docMakerName);
props.setProperty("line.file.out", file.getAbsolutePath());
props.setProperty("line.file.out", file.toAbsolutePath().toString());
props.setProperty("directory", "RAMDirectory"); // no accidental FS dir.
if (allowEmptyDocs) {
props.setProperty("sufficient.fields", ",");
@ -154,9 +154,9 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
return new PerfRunData(config);
}
private void doReadTest(File file, Type fileType, String expTitle,
private void doReadTest(Path file, Type fileType, String expTitle,
String expDate, String expBody) throws Exception {
InputStream in = new FileInputStream(file);
InputStream in = Files.newInputStream(file);
switch(fileType) {
case BZIP2:
in = csFactory.createCompressorInputStream(CompressorStreamFactory.BZIP2, in);
@ -169,8 +169,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
default:
assertFalse("Unknown file type!",true); //fail, should not happen
}
BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8));
try {
try (BufferedReader br = new BufferedReader(new InputStreamReader(in, StandardCharsets.UTF_8))) {
String line = br.readLine();
assertHeaderLine(line);
line = br.readLine();
@ -184,8 +183,6 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
assertEquals(expBody, parts[2]);
}
assertNull(br.readLine());
} finally {
br.close();
}
}
@ -197,7 +194,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
public void testBZip2() throws Exception {
// Create a document in bz2 format.
File file = new File(getWorkDir(), "one-line.bz2");
Path file = getWorkDir().resolve("one-line.bz2");
PerfRunData runData = createPerfRunData(file, false, WriteLineDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
@ -210,7 +207,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
public void testGZip() throws Exception {
// Create a document in gz format.
File file = new File(getWorkDir(), "one-line.gz");
Path file = getWorkDir().resolve("one-line.gz");
PerfRunData runData = createPerfRunData(file, false, WriteLineDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
@ -222,7 +219,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
public void testRegularFile() throws Exception {
// Create a document in regular format.
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, false, WriteLineDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
@ -235,7 +232,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
// WriteLineDocTask replaced only \t characters w/ a space, since that's its
// separator char. However, it didn't replace newline characters, which
// resulted in errors in LineDocSource.
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, false, NewLinesDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
@ -248,7 +245,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
// WriteLineDocTask threw away documents w/ no BODY element, even if they
// had a TITLE element (LUCENE-1755). It should throw away documents if they
// don't have BODY nor TITLE
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, false, NoBodyDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
@ -258,7 +255,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
}
public void testEmptyTitle() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, false, NoTitleDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
@ -269,61 +266,52 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
/** Fail by default when there's only date */
public void testJustDate() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, false, JustDateDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
wldt.close();
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8));
try {
try (BufferedReader br = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
String line = br.readLine();
assertHeaderLine(line);
line = br.readLine();
assertNull(line);
} finally {
br.close();
}
}
public void testLegalJustDate() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, false, LegalJustDateDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
wldt.close();
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8));
try {
try (BufferedReader br = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
String line = br.readLine();
assertHeaderLine(line);
line = br.readLine();
assertNotNull(line);
} finally {
br.close();
}
}
public void testEmptyDoc() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, true, EmptyDocMaker.class.getName());
WriteLineDocTask wldt = new WriteLineDocTask(runData);
wldt.doLogic();
wldt.close();
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8));
try {
try (BufferedReader br = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
String line = br.readLine();
assertHeaderLine(line);
line = br.readLine();
assertNotNull(line);
} finally {
br.close();
}
}
public void testMultiThreaded() throws Exception {
File file = new File(getWorkDir(), "one-line");
Path file = getWorkDir().resolve("one-line");
PerfRunData runData = createPerfRunData(file, false, ThreadingDocMaker.class.getName());
final WriteLineDocTask wldt = new WriteLineDocTask(runData);
Thread[] threads = new Thread[10];
@ -346,8 +334,7 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
wldt.close();
Set<String> ids = new HashSet<>();
BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(file), StandardCharsets.UTF_8));
try {
try (BufferedReader br = Files.newBufferedReader(file, StandardCharsets.UTF_8)) {
String line = br.readLine();
assertHeaderLine(line); // header line is written once, no matter how many threads there are
for (int i = 0; i < threads.length; i++) {
@ -363,9 +350,6 @@ public class WriteLineDocTaskTest extends BenchmarkTestCase {
// only threads.length lines should exist
assertNull(br.readLine());
assertEquals(threads.length, ids.size());
} finally {
br.close();
}
}
}

37
lucene/benchmark/src/test/org/apache/lucene/benchmark/byTask/utils/StreamUtilsTest.java
View File

@ -19,26 +19,25 @@ package org.apache.lucene.benchmark.byTask.utils;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.io.OutputStreamWriter;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import org.apache.commons.compress.compressors.CompressorStreamFactory;
import org.apache.lucene.benchmark.BenchmarkTestCase;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.TestUtil;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
public class StreamUtilsTest extends BenchmarkTestCase {
private static final String TEXT = "Some-Text...";
private File testDir;
private Path testDir;
@Test
public void testGetInputStreamPlainText() throws Exception {
@ -86,31 +85,31 @@ public class StreamUtilsTest extends BenchmarkTestCase {
assertReadText(autoOutFile("TEXT"));
}
private File rawTextFile(String ext) throws Exception {
File f = new File(testDir,"testfile." + ext);
BufferedWriter w = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), StandardCharsets.UTF_8));
private Path rawTextFile(String ext) throws Exception {
Path f = testDir.resolve("testfile." + ext);
BufferedWriter w = Files.newBufferedWriter(f, StandardCharsets.UTF_8);
w.write(TEXT);
w.newLine();
w.close();
return f;
}
private File rawGzipFile(String ext) throws Exception {
File f = new File(testDir,"testfile." + ext);
OutputStream os = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP, new FileOutputStream(f));
private Path rawGzipFile(String ext) throws Exception {
Path f = testDir.resolve("testfile." + ext);
OutputStream os = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.GZIP, Files.newOutputStream(f));
writeText(os);
return f;
}
private File rawBzip2File(String ext) throws Exception {
File f = new File(testDir,"testfile." + ext);
OutputStream os = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.BZIP2, new FileOutputStream(f));
private Path rawBzip2File(String ext) throws Exception {
Path f = testDir.resolve("testfile." + ext);
OutputStream os = new CompressorStreamFactory().createCompressorOutputStream(CompressorStreamFactory.BZIP2, Files.newOutputStream(f));
writeText(os);
return f;
}
private File autoOutFile(String ext) throws Exception {
File f = new File(testDir,"testfile." + ext);
private Path autoOutFile(String ext) throws Exception {
Path f = testDir.resolve("testfile." + ext);
OutputStream os = StreamUtils.outputStream(f);
writeText(os);
return f;
@ -123,12 +122,12 @@ public class StreamUtilsTest extends BenchmarkTestCase {
w.close();
}
private void assertReadText(File f) throws Exception {
private void assertReadText(Path f) throws Exception {
InputStream ir = StreamUtils.inputStream(f);
InputStreamReader in = new InputStreamReader(ir, StandardCharsets.UTF_8);
BufferedReader r = new BufferedReader(in);
String line = r.readLine();
assertEquals("Wrong text found in "+f.getName(), TEXT, line);
assertEquals("Wrong text found in "+f.getFileName(), TEXT, line);
r.close();
}
@ -136,9 +135,9 @@ public class StreamUtilsTest extends BenchmarkTestCase {
@Before
public void setUp() throws Exception {
super.setUp();
testDir = new File(getWorkDir(),"ContentSourceTest");
testDir = getWorkDir().resolve("ContentSourceTest");
IOUtils.rm(testDir);
assertTrue(testDir.mkdirs());
Files.createDirectory(testDir);
}
@Override

4
lucene/benchmark/src/test/org/apache/lucene/benchmark/quality/TestQualityRun.java
View File

@ -26,10 +26,8 @@ import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase.SuppressSysoutChecks;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@ -73,7 +71,7 @@ public class TestQualityRun extends BenchmarkTestCase {
// validate topics & judgments match each other
judge.validateData(qqs, logger);
Directory dir = newFSDirectory(new File(getWorkDir(),"index"));
Directory dir = newFSDirectory(getWorkDir().resolve("index"));
IndexReader reader = DirectoryReader.open(dir);
IndexSearcher searcher = new IndexSearcher(reader);

19
lucene/common-build.xml
View File

@ -2356,7 +2356,16 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
</path>
</target>
<target name="check-forbidden-apis" depends="-check-forbidden-all,-check-forbidden-core,-check-forbidden-tests" description="Check forbidden API calls in compiled class files"/>
<condition property="islucene" value="true" else="false">
<not>
<or>
<matches pattern="^(solr)\b" string="${name}"/>
<matches pattern="tools" string="${name}"/>
</or>
</not>
</condition>
<target name="check-forbidden-apis" depends="-check-forbidden-all,-check-forbidden-core,-check-forbidden-tests,-check-forbidden-lucene" description="Check forbidden API calls in compiled class files"/>
<!-- applies to both source and test code -->
<target name="-check-forbidden-all" depends="-init-forbidden-apis,compile-core,compile-test">
@ -2368,6 +2377,14 @@ ${ant.project.name}.test.dependencies=${test.classpath.list}
<fileset dir="${build.dir}/classes/test" excludes="${forbidden-tests-excludes}" erroronmissingdir="false"/>
</forbidden-apis>
</target>
<!-- applies to only lucene API -->
<target name="-check-forbidden-lucene" depends="-init-forbidden-apis,compile-core,compile-test" if="${islucene}">
<forbidden-apis signaturesFile="${common.dir}/tools/forbiddenApis/lucene.txt" classpathref="forbidden-apis.allclasses.classpath">
<fileset dir="${build.dir}/classes/java" excludes="${forbidden-base-excludes}"/>
<fileset dir="${build.dir}/classes/test" excludes="${forbidden-tests-excludes}" erroronmissingdir="false"/>
</forbidden-apis>
</target>
<!-- applies to only test code -->
<target name="-check-forbidden-tests" depends="-init-forbidden-apis,compile-test">

5
lucene/core/build.xml
View File

@ -34,6 +34,11 @@
org/apache/lucene/util/RamUsageEstimator.class
"/>
<!-- TODO: maybe let people get closedchannel if they cancel(true) -->
<property name="forbidden-base-excludes" value="
org/apache/lucene/store/SimpleFSDirectory.class
"/>
<import file="../common-build.xml"/>
<property name="moman.commit-hash" value="5c5c2a1e4dea" />

8
lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -17,9 +17,10 @@ package org.apache.lucene.index;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.HashMap;
@ -2046,11 +2047,12 @@ public class CheckIndex {
System.out.println("\nOpening index @ " + indexPath + "\n");
Directory dir = null;
Path path = Paths.get(indexPath);
try {
if (dirImpl == null) {
dir = FSDirectory.open(new File(indexPath));
dir = FSDirectory.open(path);
} else {
dir = CommandLineUtil.newFSDirectory(dirImpl, new File(indexPath));
dir = CommandLineUtil.newFSDirectory(dirImpl, path);
}
} catch (Throwable t) {
System.out.println("ERROR: could not open directory \"" + indexPath + "\"; exiting");

21
lucene/core/src/java/org/apache/lucene/index/DirectoryReader.java
View File

@ -26,7 +26,6 @@ import java.util.List;
import org.apache.lucene.search.SearcherManager; // javadocs
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NoSuchDirectoryException;
/** DirectoryReader is an implementation of {@link CompositeReader}
that can read indexes in a {@link Directory}.
@ -288,22 +287,12 @@ public abstract class DirectoryReader extends BaseCompositeReader<AtomicReader>
// corrupt indices. This means that IndexWriter will
// throw an exception on such indices and the app must
// resolve the situation manually:
String[] files;
try {
files = directory.listAll();
} catch (NoSuchDirectoryException nsde) {
// Directory does not exist --> no index exists
return false;
}
String[] files = directory.listAll();
// Defensive: maybe a Directory impl returns null
// instead of throwing NoSuchDirectoryException:
if (files != null) {
String prefix = IndexFileNames.SEGMENTS + "_";
for(String file : files) {
if (file.startsWith(prefix)) {
return true;
}
String prefix = IndexFileNames.SEGMENTS + "_";
for(String file : files) {
if (file.startsWith(prefix)) {
return true;
}
}
return false;

9
lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
View File

@ -31,7 +31,6 @@ import java.util.regex.Matcher;
import org.apache.lucene.store.AlreadyClosedException;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.CollectionUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.InfoStream;
@ -144,13 +143,7 @@ final class IndexFileDeleter implements Closeable {
long currentGen = segmentInfos.getGeneration();
CommitPoint currentCommitPoint = null;
String[] files = null;
try {
files = directory.listAll();
} catch (NoSuchDirectoryException e) {
// it means the directory is empty, so ignore it.
files = new String[0];
}
String[] files = directory.listAll();
if (currentSegmentsFile != null) {
Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");

8
lucene/core/src/java/org/apache/lucene/index/IndexUpgrader.java
View File

@ -24,8 +24,9 @@ import org.apache.lucene.util.InfoStream;
import org.apache.lucene.util.PrintStreamInfoStream;
import org.apache.lucene.util.Version;
import java.io.File;
import java.io.IOException;
import java.nio.file.Paths;
import java.nio.file.Path;
import java.util.Collection;
/**
@ -102,11 +103,12 @@ public final class IndexUpgrader {
printUsage();
}
Path p = Paths.get(path);
Directory dir = null;
if (dirImpl == null) {
dir = FSDirectory.open(new File(path));
dir = FSDirectory.open(p);
} else {
dir = CommandLineUtil.newFSDirectory(dirImpl, new File(path));
dir = CommandLineUtil.newFSDirectory(dirImpl, p);
}
return new IndexUpgrader(dir, out, deletePriorCommits);
}

11
lucene/core/src/java/org/apache/lucene/index/SegmentInfos.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import java.io.PrintStream;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@ -41,7 +42,6 @@ import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
import org.apache.lucene.store.NoSuchDirectoryException;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.StringHelper;
@ -176,9 +176,6 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
* @param files -- array of file names to check
*/
public static long getLastCommitGeneration(String[] files) {
if (files == null) {
return -1;
}
long max = -1;
for (String file : files) {
if (file.startsWith(IndexFileNames.SEGMENTS) && !file.equals(IndexFileNames.OLD_SEGMENTS_GEN)) {
@ -198,11 +195,7 @@ public final class SegmentInfos implements Cloneable, Iterable<SegmentCommitInfo
* @param directory -- directory to search for the latest segments_N file
*/
public static long getLastCommitGeneration(Directory directory) throws IOException {
try {
return getLastCommitGeneration(directory.listAll());
} catch (NoSuchDirectoryException nsde) {
return -1;
}
return getLastCommitGeneration(directory.listAll());
}
/**

4
lucene/core/src/java/org/apache/lucene/store/Directory.java
View File

@ -46,9 +46,7 @@ public abstract class Directory implements Closeable {
/**
* Returns an array of strings, one for each file in the directory.
*
* @throws NoSuchDirectoryException if the directory is not prepared for any
* write operations (such as {@link #createOutput(String, IOContext)}).
* @throws IOException in case of other IO errors
* @throws IOException in case of IO error
*/
public abstract String[] listAll() throws IOException;

110
lucene/core/src/java/org/apache/lucene/store/FSDirectory.java
View File

@ -20,16 +20,17 @@ package org.apache.lucene.store;
import org.apache.lucene.util.Constants;
import org.apache.lucene.util.IOUtils;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FilenameFilter;
import java.io.FilterOutputStream;
import java.io.IOException;
import java.nio.file.DirectoryStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
import java.util.concurrent.Future;
@ -114,32 +115,25 @@ import static java.util.Collections.synchronizedSet;
*/
public abstract class FSDirectory extends BaseDirectory {
protected final File directory; // The underlying filesystem directory
protected final Path directory; // The underlying filesystem directory
protected final Set<String> staleFiles = synchronizedSet(new HashSet<String>()); // Files written, but not yet sync'ed
// returns the canonical version of the directory, creating it if it doesn't exist.
private static File getCanonicalPath(File file) throws IOException {
return new File(file.getCanonicalPath());
}
/** Create a new FSDirectory for the named location (ctor for subclasses).
* @param path the path of the directory
* @param lockFactory the lock factory to use, or null for the default
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error
*/
protected FSDirectory(File path, LockFactory lockFactory) throws IOException {
protected FSDirectory(Path path, LockFactory lockFactory) throws IOException {
// new ctors use always NativeFSLockFactory as default:
if (lockFactory == null) {
lockFactory = new NativeFSLockFactory();
}
directory = getCanonicalPath(path);
if (directory.exists() && !directory.isDirectory())
throw new NoSuchDirectoryException("file '" + directory + "' exists but is not a directory");
Files.createDirectories(path); // create directory, if it doesnt exist
directory = path.toRealPath();
setLockFactory(lockFactory);
}
/** Creates an FSDirectory instance, trying to pick the
@ -162,13 +156,13 @@ public abstract class FSDirectory extends BaseDirectory {
* {@link MMapDirectory} on 64 bit JVMs.
*
* <p>See <a href="#subclasses">above</a> */
public static FSDirectory open(File path) throws IOException {
public static FSDirectory open(Path path) throws IOException {
return open(path, null);
}
/** Just like {@link #open(File)}, but allows you to
/** Just like {@link #open(Path)}, but allows you to
* also specify a custom {@link LockFactory}. */
public static FSDirectory open(File path, LockFactory lockFactory) throws IOException {
public static FSDirectory open(Path path, LockFactory lockFactory) throws IOException {
if (Constants.JRE_IS_64BIT && MMapDirectory.UNMAP_SUPPORTED) {
return new MMapDirectory(path, lockFactory);
} else if (Constants.WINDOWS) {
@ -186,12 +180,12 @@ public abstract class FSDirectory extends BaseDirectory {
// in index dir. If no index dir is given, set ourselves
if (lockFactory instanceof FSLockFactory) {
final FSLockFactory lf = (FSLockFactory) lockFactory;
final File dir = lf.getLockDir();
final Path dir = lf.getLockDir();
// if the lock factory has no lockDir set, use the this directory as lockDir
if (dir == null) {
lf.setLockDir(directory);
lf.setLockPrefix(null);
} else if (dir.getCanonicalPath().equals(directory.getCanonicalPath())) {
} else if (dir.toRealPath().equals(directory)) {
lf.setLockPrefix(null);
}
}
@ -199,36 +193,29 @@ public abstract class FSDirectory extends BaseDirectory {
}
/** Lists all files (not subdirectories) in the
* directory. This method never returns null (throws
* {@link IOException} instead).
* directory.
*
* @throws NoSuchDirectoryException if the directory
* does not exist, or does exist but is not a
* directory.
* @throws IOException if list() returns null */
public static String[] listAll(File dir) throws IOException {
if (!dir.exists())
throw new NoSuchDirectoryException("directory '" + dir + "' does not exist");
else if (!dir.isDirectory())
throw new NoSuchDirectoryException("file '" + dir + "' exists but is not a directory");
// Exclude subdirs
String[] result = dir.list(new FilenameFilter() {
@Override
public boolean accept(File dir, String file) {
return !new File(dir, file).isDirectory();
}
});
if (result == null)
throw new IOException("directory '" + dir + "' exists and is a directory, but cannot be listed: list() returned null");
return result;
* @throws IOException if there was an I/O error during listing */
public static String[] listAll(Path dir) throws IOException {
List<String> entries = new ArrayList<>();
try (DirectoryStream<Path> stream = Files.newDirectoryStream(dir, new DirectoryStream.Filter<Path>() {
@Override
public boolean accept(Path entry) throws IOException {
return !Files.isDirectory(entry); // filter out entries that are definitely directories.
}
})) {
for (Path path : stream) {
entries.add(path.getFileName().toString());
}
}
return entries.toArray(new String[entries.size()]);
}
/** Lists all files (not subdirectories) in the
* directory.
* @see #listAll(File) */
* @see #listAll(Path) */
@Override
public String[] listAll() throws IOException {
ensureOpen();
@ -239,21 +226,14 @@ public abstract class FSDirectory extends BaseDirectory {
@Override
public long fileLength(String name) throws IOException {
ensureOpen();
File file = new File(directory, name);
final long len = file.length();
if (len == 0 && !file.exists()) {
throw new FileNotFoundException(name);
} else {
return len;
}
return Files.size(directory.resolve(name));
}
/** Removes an existing file in the directory. */
@Override
public void deleteFile(String name) throws IOException {
ensureOpen();
File file = new File(directory, name);
Files.delete(file.toPath());
Files.delete(directory.resolve(name));
staleFiles.remove(name);
}
@ -267,12 +247,7 @@ public abstract class FSDirectory extends BaseDirectory {
}
protected void ensureCanWrite(String name) throws IOException {
if (!directory.exists())
if (!directory.mkdirs())
throw new IOException("Cannot create directory: " + directory);
File file = new File(directory, name);
Files.deleteIfExists(file.toPath()); // delete existing, if any
Files.deleteIfExists(directory.resolve(name)); // delete existing, if any
}
/**
@ -299,7 +274,7 @@ public abstract class FSDirectory extends BaseDirectory {
@Override
public void renameFile(String source, String dest) throws IOException {
ensureOpen();
Files.move(new File(directory, source).toPath(), new File(directory, dest).toPath(), StandardCopyOption.ATOMIC_MOVE);
Files.move(directory.resolve(source), directory.resolve(dest), StandardCopyOption.ATOMIC_MOVE);
// TODO: should we move directory fsync to a separate 'syncMetadata' method?
// for example, to improve listCommits(), IndexFileDeleter could also call that after deleting segments_Ns
IOUtils.fsync(directory, true);
@ -308,12 +283,7 @@ public abstract class FSDirectory extends BaseDirectory {
@Override
public String getLockID() {
ensureOpen();
String dirName; // name to be hashed
try {
dirName = directory.getCanonicalPath();
} catch (IOException e) {
throw new RuntimeException(e.toString(), e);
}
String dirName = directory.toString(); // name to be hashed
int digest = 0;
for(int charIDX=0;charIDX<dirName.length();charIDX++) {
@ -330,7 +300,7 @@ public abstract class FSDirectory extends BaseDirectory {
}
/** @return the underlying filesystem directory */
public File getDirectory() {
public Path getDirectory() {
ensureOpen();
return directory;
}
@ -351,7 +321,7 @@ public abstract class FSDirectory extends BaseDirectory {
private final String name;
public FSIndexOutput(String name) throws IOException {
super(new FilterOutputStream(new FileOutputStream(new File(directory, name))) {
super(new FilterOutputStream(Files.newOutputStream(directory.resolve(name))) {
// This implementation ensures, that we never write more than CHUNK_SIZE bytes:
@Override
public void write(byte[] b, int offset, int length) throws IOException {
@ -377,6 +347,6 @@ public abstract class FSDirectory extends BaseDirectory {
}
protected void fsync(String name) throws IOException {
IOUtils.fsync(new File(directory, name), false);
IOUtils.fsync(directory.resolve(name), false);
}
}

13
lucene/core/src/java/org/apache/lucene/store/FSLockFactory.java
View File

@ -17,7 +17,9 @@ package org.apache.lucene.store;
* limitations under the License.
*/
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* Base class for file system based locking implementation.
@ -28,7 +30,7 @@ public abstract class FSLockFactory extends LockFactory {
/**
* Directory for the lock files.
*/
protected File lockDir = null;
protected Path lockDir = null;
/**
* Set the lock directory. This method can be only called
@ -37,16 +39,19 @@ public abstract class FSLockFactory extends LockFactory {
* Subclasses can also use this method to set the directory
* in the constructor.
*/
protected final void setLockDir(File lockDir) {
protected final void setLockDir(Path lockDir) throws IOException {
if (this.lockDir != null)
throw new IllegalStateException("You can set the lock directory for this factory only once.");
if (lockDir != null) {
Files.createDirectories(lockDir);
}
this.lockDir = lockDir;
}
/**
* Retrieve the lock directory.
*/
public File getLockDir() {
public Path getLockDir() {
return lockDir;
}

15
lucene/core/src/java/org/apache/lucene/store/FileSwitchDirectory.java
View File

@ -19,6 +19,7 @@ package org.apache.lucene.store;
import java.io.IOException;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.NoSuchFileException;
import java.util.ArrayList;
import java.util.Collection;
@ -82,32 +83,32 @@ public class FileSwitchDirectory extends BaseDirectory {
// LUCENE-3380: either or both of our dirs could be FSDirs,
// but if one underlying delegate is an FSDir and mkdirs() has not
// yet been called, because so far everything is written to the other,
// in this case, we don't want to throw a NoSuchDirectoryException
NoSuchDirectoryException exc = null;
// in this case, we don't want to throw a NoSuchFileException
NoSuchFileException exc = null;
try {
for(String f : primaryDir.listAll()) {
files.add(f);
}
} catch (NoSuchDirectoryException e) {
} catch (NoSuchFileException e) {
exc = e;
}
try {
for(String f : secondaryDir.listAll()) {
files.add(f);
}
} catch (NoSuchDirectoryException e) {
// we got NoSuchDirectoryException from both dirs
} catch (NoSuchFileException e) {
// we got NoSuchFileException from both dirs
// rethrow the first.
if (exc != null) {
throw exc;
}
// we got NoSuchDirectoryException from the secondary,
// we got NoSuchFileException from the secondary,
// and the primary is empty.
if (files.isEmpty()) {
throw e;
}
}
// we got NoSuchDirectoryException from the primary,
// we got NoSuchFileException from the primary,
// and the secondary is empty.
if (exc != null && files.isEmpty()) {
throw exc;

5
lucene/core/src/java/org/apache/lucene/store/LockStressTest.java
View File

@ -18,11 +18,12 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
import java.io.File;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.InetSocketAddress;
import java.net.Socket;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Random;
/**
@ -133,7 +134,7 @@ public class LockStressTest {
throw new IOException("Cannot instantiate lock factory " + lockFactoryClassName);
}
File lockDir = new File(lockDirName);
Path lockDir = Paths.get(lockDirName);
if (lockFactory instanceof FSLockFactory) {
((FSLockFactory) lockFactory).setLockDir(lockDir);

20
lucene/core/src/java/org/apache/lucene/store/MMapDirectory.java
View File

@ -18,11 +18,11 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
import java.io.File;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.security.AccessController;
@ -44,7 +44,7 @@ import org.apache.lucene.util.Constants;
* be sure your have plenty of virtual address space, e.g. by
* using a 64 bit JRE, or a 32 bit JRE with indexes that are
* guaranteed to fit within the address space.
* On 32 bit platforms also consult {@link #MMapDirectory(File, LockFactory, int)}
* On 32 bit platforms also consult {@link #MMapDirectory(Path, LockFactory, int)}
* if you have problems with mmap failing because of fragmented
* address space. If you get an OutOfMemoryException, it is recommended
* to reduce the chunk size, until it works.
@ -83,7 +83,7 @@ public class MMapDirectory extends FSDirectory {
private boolean useUnmapHack = UNMAP_SUPPORTED;
/**
* Default max chunk size.
* @see #MMapDirectory(File, LockFactory, int)
* @see #MMapDirectory(Path, LockFactory, int)
*/
public static final int DEFAULT_MAX_BUFF = Constants.JRE_IS_64BIT ? (1 << 30) : (1 << 28);
final int chunkSizePower;
@ -95,7 +95,7 @@ public class MMapDirectory extends FSDirectory {
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error
*/
public MMapDirectory(File path, LockFactory lockFactory) throws IOException {
public MMapDirectory(Path path, LockFactory lockFactory) throws IOException {
this(path, lockFactory, DEFAULT_MAX_BUFF);
}
@ -104,7 +104,7 @@ public class MMapDirectory extends FSDirectory {
* @param path the path of the directory
* @throws IOException if there is a low-level I/O error
*/
public MMapDirectory(File path) throws IOException {
public MMapDirectory(Path path) throws IOException {
this(path, null);
}
@ -128,7 +128,7 @@ public class MMapDirectory extends FSDirectory {
* <b>Please note:</b> The chunk size is always rounded down to a power of 2.
* @throws IOException if there is a low-level I/O error
*/
public MMapDirectory(File path, LockFactory lockFactory, int maxChunkSize) throws IOException {
public MMapDirectory(Path path, LockFactory lockFactory, int maxChunkSize) throws IOException {
super(path, lockFactory);
if (maxChunkSize <= 0) {
throw new IllegalArgumentException("Maximum chunk size for mmap must be >0");
@ -182,7 +182,7 @@ public class MMapDirectory extends FSDirectory {
/**
* Returns the current mmap chunk size.
* @see #MMapDirectory(File, LockFactory, int)
* @see #MMapDirectory(Path, LockFactory, int)
*/
public final int getMaxChunkSize() {
return 1 << chunkSizePower;
@ -192,9 +192,9 @@ public class MMapDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
File file = new File(getDirectory(), name);
try (FileChannel c = FileChannel.open(file.toPath(), StandardOpenOption.READ)) {
final String resourceDescription = "MMapIndexInput(path=\"" + file.toString() + "\")";
Path path = directory.resolve(name);
try (FileChannel c = FileChannel.open(path, StandardOpenOption.READ)) {
final String resourceDescription = "MMapIndexInput(path=\"" + path.toString() + "\")";
final boolean useUnmap = getUseUnmap();
return ByteBufferIndexInput.newInstance(resourceDescription,
map(resourceDescription, c, 0, c.size()),

10
lucene/core/src/java/org/apache/lucene/store/NIOFSDirectory.java
View File

@ -17,12 +17,12 @@ package org.apache.lucene.store;
* the License.
*/
import java.io.File;
import java.io.EOFException;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.ClosedChannelException; // javadoc @link
import java.nio.channels.FileChannel;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.concurrent.Future; // javadoc
@ -60,7 +60,7 @@ public class NIOFSDirectory extends FSDirectory {
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error
*/
public NIOFSDirectory(File path, LockFactory lockFactory) throws IOException {
public NIOFSDirectory(Path path, LockFactory lockFactory) throws IOException {
super(path, lockFactory);
}
@ -69,7 +69,7 @@ public class NIOFSDirectory extends FSDirectory {
* @param path the path of the directory
* @throws IOException if there is a low-level I/O error
*/
public NIOFSDirectory(File path) throws IOException {
public NIOFSDirectory(Path path) throws IOException {
super(path, null);
}
@ -77,8 +77,8 @@ public class NIOFSDirectory extends FSDirectory {
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
File path = new File(getDirectory(), name);
FileChannel fc = FileChannel.open(path.toPath(), StandardOpenOption.READ);
Path path = getDirectory().resolve(name);
FileChannel fc = FileChannel.open(path, StandardOpenOption.READ);
return new NIOFSIndexInput("NIOFSIndexInput(path=\"" + path + "\")", fc, context);
}

24
lucene/core/src/java/org/apache/lucene/store/NRTCachingDirectory.java
View File

@ -18,11 +18,11 @@ package org.apache.lucene.store;
*/
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.index.IndexFileNames;
import org.apache.lucene.store.RAMDirectory; // javadocs
import org.apache.lucene.util.Accountable;
import org.apache.lucene.util.IOUtils;
@ -49,7 +49,7 @@ import org.apache.lucene.util.IOUtils;
* <p>Here's a simple example usage:
*
* <pre class="prettyprint">
* Directory fsDir = FSDirectory.open(new File("/path/to/index"));
* Directory fsDir = FSDirectory.open(new File("/path/to/index").toPath());
* NRTCachingDirectory cachedFSDir = new NRTCachingDirectory(fsDir, 5.0, 60.0);
* IndexWriterConfig conf = new IndexWriterConfig(analyzer);
* IndexWriter writer = new IndexWriter(cachedFSDir, conf);
@ -96,22 +96,10 @@ public class NRTCachingDirectory extends FilterDirectory implements Accountable
for(String f : cache.listAll()) {
files.add(f);
}
// LUCENE-1468: our NRTCachingDirectory will actually exist (RAMDir!),
// but if the underlying delegate is an FSDir and mkdirs() has not
// yet been called, because so far everything is a cached write,
// in this case, we don't want to throw a NoSuchDirectoryException
try {
for(String f : in.listAll()) {
// Cannot do this -- if lucene calls createOutput but
// file already exists then this falsely trips:
//assert !files.contains(f): "file \"" + f + "\" is in both dirs";
files.add(f);
}
} catch (NoSuchDirectoryException ex) {
// however, if there are no cached files, then the directory truly
// does not "exist"
if (files.isEmpty()) {
throw ex;
for(String f : in.listAll()) {
if (!files.add(f)) {
throw new IllegalStateException("file: " + in + " appears both in delegate and in cache: " +
"cache=" + Arrays.toString(cache.listAll()) + ",delegate=" + Arrays.toString(in.listAll()));
}
}
return files.toArray(new String[files.size()]);

55
lucene/core/src/java/org/apache/lucene/store/NativeFSLockFactory.java
View File

@ -20,6 +20,8 @@ package org.apache.lucene.store;
import java.nio.channels.FileChannel;
import java.nio.channels.FileLock;
import java.nio.channels.OverlappingFileLockException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.io.File;
import java.io.IOException;
@ -77,18 +79,8 @@ public class NativeFSLockFactory extends FSLockFactory {
* directory itself. Be sure to create one instance for each directory
* your create!
*/
public NativeFSLockFactory() {
this((File) null);
}
/**
* Create a NativeFSLockFactory instance, storing lock
* files into the specified lockDirName:
*
* @param lockDirName where lock files are created.
*/
public NativeFSLockFactory(String lockDirName) {
this(new File(lockDirName));
public NativeFSLockFactory() throws IOException {
this((Path) null);
}
/**
@ -97,7 +89,7 @@ public class NativeFSLockFactory extends FSLockFactory {
*
* @param lockDir where lock files are created.
*/
public NativeFSLockFactory(File lockDir) {
public NativeFSLockFactory(Path lockDir) throws IOException {
setLockDir(lockDir);
}
@ -118,14 +110,14 @@ class NativeFSLock extends Lock {
private FileChannel channel;
private FileLock lock;
private File path;
private File lockDir;
private Path path;
private Path lockDir;
private static final Set<String> LOCK_HELD = Collections.synchronizedSet(new HashSet<String>());
public NativeFSLock(File lockDir, String lockFileName) {
public NativeFSLock(Path lockDir, String lockFileName) {
this.lockDir = lockDir;
path = new File(lockDir, lockFileName);
path = lockDir.resolve(lockFileName);
}
@ -138,16 +130,14 @@ class NativeFSLock extends Lock {
}
// Ensure that lockDir exists and is a directory.
if (!lockDir.exists()) {
if (!lockDir.mkdirs())
throw new IOException("Cannot create directory: " +
lockDir.getAbsolutePath());
} else if (!lockDir.isDirectory()) {
// TODO: NoSuchDirectoryException instead?
throw new IOException("Found regular file where directory expected: " +
lockDir.getAbsolutePath());
Files.createDirectories(lockDir);
try {
Files.createFile(path);
} catch (IOException ignore) {
// we must create the file to have a truly canonical path.
// if its already created, we don't care. if it cant be created, it will fail below.
}
final String canonicalPath = path.getCanonicalPath();
final Path canonicalPath = path.toRealPath();
// Make sure nobody else in-process has this lock held
// already, and, mark it held if not:
// This is a pretty crazy workaround for some documented
@ -162,9 +152,9 @@ class NativeFSLock extends Lock {
// is that we can't re-obtain the lock in the same JVM but from a different process if that happens. Nevertheless
// this is super trappy. See LUCENE-5738
boolean obtained = false;
if (LOCK_HELD.add(canonicalPath)) {
if (LOCK_HELD.add(canonicalPath.toString())) {
try {
channel = FileChannel.open(path.toPath(), StandardOpenOption.CREATE, StandardOpenOption.WRITE);
channel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
try {
lock = channel.tryLock();
obtained = lock != null;
@ -209,8 +199,9 @@ class NativeFSLock extends Lock {
}
}
private static final void clearLockHeld(File path) throws IOException {
boolean remove = LOCK_HELD.remove(path.getCanonicalPath());
private static final void clearLockHeld(Path path) throws IOException {
path = path.toRealPath();
boolean remove = LOCK_HELD.remove(path.toString());
assert remove : "Lock was cleared but never marked as held";
}
@ -221,8 +212,8 @@ class NativeFSLock extends Lock {
// First a shortcut, if a lock reference in this instance is available
if (lock != null) return true;
// Look if lock file is present; if not, there can definitely be no lock!
if (!path.exists()) return false;
// Look if lock file is definitely not present; if not, there can definitely be no lock!
if (Files.notExists(path)) return false;
// Try to obtain and release (if was locked) the lock
try {

31
lucene/core/src/java/org/apache/lucene/store/NoSuchDirectoryException.java
View File

@ -1,31 +0,0 @@
package org.apache.lucene.store;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.io.FileNotFoundException;
/**
* This exception is thrown when you try to list a
* non-existent directory.
*/
public class NoSuchDirectoryException extends FileNotFoundException {
public NoSuchDirectoryException(String message) {
super(message);
}
}

11
lucene/core/src/java/org/apache/lucene/store/SimpleFSDirectory.java
View File

@ -21,6 +21,7 @@ import java.io.EOFException;
import java.io.File;
import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.file.Path;
/** A straightforward implementation of {@link FSDirectory}
* using java.io.RandomAccessFile. However, this class has
@ -28,6 +29,8 @@ import java.io.RandomAccessFile;
* bottleneck) as it synchronizes when multiple threads
* read from the same file. It's usually better to use
* {@link NIOFSDirectory} or {@link MMapDirectory} instead. */
// TODO: we currently mandate .toFile to still use RandomAccessFile, to avoid ClosedByInterruptException.
// should we change to SeekableByteChannel instead?
public class SimpleFSDirectory extends FSDirectory {
/** Create a new SimpleFSDirectory for the named location.
@ -37,8 +40,9 @@ public class SimpleFSDirectory extends FSDirectory {
* ({@link NativeFSLockFactory});
* @throws IOException if there is a low-level I/O error
*/
public SimpleFSDirectory(File path, LockFactory lockFactory) throws IOException {
public SimpleFSDirectory(Path path, LockFactory lockFactory) throws IOException {
super(path, lockFactory);
path.toFile(); // throw exception if we can't get a File for now
}
/** Create a new SimpleFSDirectory for the named location and {@link NativeFSLockFactory}.
@ -46,15 +50,16 @@ public class SimpleFSDirectory extends FSDirectory {
* @param path the path of the directory
* @throws IOException if there is a low-level I/O error
*/
public SimpleFSDirectory(File path) throws IOException {
public SimpleFSDirectory(Path path) throws IOException {
super(path, null);
path.toFile(); // throw exception if we can't get a File for now
}
/** Creates an IndexInput for the file with the given name. */
@Override
public IndexInput openInput(String name, IOContext context) throws IOException {
ensureOpen();
final File path = new File(directory, name);
final File path = directory.resolve(name).toFile();
RandomAccessFile raf = new RandomAccessFile(path, "r");
return new SimpleFSIndexInput("SimpleFSIndexInput(path=\"" + path.getPath() + "\")", raf, context);
}

56
lucene/core/src/java/org/apache/lucene/store/SimpleFSLockFactory.java
View File

@ -20,10 +20,11 @@ package org.apache.lucene.store;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
/**
* <p>Implements {@link LockFactory} using {@link
* File#createNewFile()}.</p>
* Files#createFile}.</p>
*
* <p><b>NOTE:</b> the {@linkplain File#createNewFile() javadocs
* for <code>File.createNewFile()</code>} contain a vague
@ -68,26 +69,18 @@ public class SimpleFSLockFactory extends FSLockFactory {
* directory itself. Be sure to create one instance for each directory
* your create!
*/
public SimpleFSLockFactory() {
this((File) null);
public SimpleFSLockFactory() throws IOException {
this((Path) null);
}
/**
* Instantiate using the provided directory (as a File instance).
* Instantiate using the provided directory (as a Path instance).
* @param lockDir where lock files should be created.
*/
public SimpleFSLockFactory(File lockDir) {
public SimpleFSLockFactory(Path lockDir) throws IOException {
setLockDir(lockDir);
}
/**
* Instantiate using the provided directory name (String).
* @param lockDirName where lock files should be created.
*/
public SimpleFSLockFactory(String lockDirName) {
setLockDir(new File(lockDirName));
}
@Override
public Lock makeLock(String lockName) {
if (lockPrefix != null) {
@ -98,42 +91,29 @@ public class SimpleFSLockFactory extends FSLockFactory {
@Override
public void clearLock(String lockName) throws IOException {
if (lockDir.exists()) {
if (lockPrefix != null) {
lockName = lockPrefix + "-" + lockName;
}
File lockFile = new File(lockDir, lockName);
Files.deleteIfExists(lockFile.toPath());
if (lockPrefix != null) {
lockName = lockPrefix + "-" + lockName;
}
Files.deleteIfExists(lockDir.resolve(lockName));
}
}
class SimpleFSLock extends Lock {
File lockFile;
File lockDir;
Path lockFile;
Path lockDir;
public SimpleFSLock(File lockDir, String lockFileName) {
public SimpleFSLock(Path lockDir, String lockFileName) {
this.lockDir = lockDir;
lockFile = new File(lockDir, lockFileName);
lockFile = lockDir.resolve(lockFileName);
}
@Override
public boolean obtain() throws IOException {
// Ensure that lockDir exists and is a directory:
if (!lockDir.exists()) {
if (!lockDir.mkdirs())
throw new IOException("Cannot create directory: " +
lockDir.getAbsolutePath());
} else if (!lockDir.isDirectory()) {
// TODO: NoSuchDirectoryException instead?
throw new IOException("Found regular file where directory expected: " +
lockDir.getAbsolutePath());
}
try {
return lockFile.createNewFile();
Files.createDirectories(lockDir);
Files.createFile(lockFile);
return true;
} catch (IOException ioe) {
// On Windows, on concurrent createNewFile, the 2nd process gets "access denied".
// In that case, the lock was not aquired successfully, so return false.
@ -148,7 +128,7 @@ class SimpleFSLock extends Lock {
public void close() throws LockReleaseFailedException {
// TODO: wierd that clearLock() throws the raw IOException...
try {
Files.deleteIfExists(lockFile.toPath());
Files.deleteIfExists(lockFile);
} catch (Throwable cause) {
throw new LockReleaseFailedException("failed to delete " + lockFile, cause);
}
@ -156,7 +136,7 @@ class SimpleFSLock extends Lock {
@Override
public boolean isLocked() {
return lockFile.exists();
return Files.exists(lockFile);
}
@Override

22
lucene/core/src/java/org/apache/lucene/util/CommandLineUtil.java
View File

@ -17,9 +17,9 @@ package org.apache.lucene.util;
* limitations under the License.
*/
import java.io.File;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Path;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@ -37,13 +37,13 @@ public final class CommandLineUtil {
/**
* Creates a specific FSDirectory instance starting from its class name
* @param clazzName The name of the FSDirectory class to load
* @param file The file to be used as parameter constructor
* @param path The path to be used as parameter constructor
* @return the new FSDirectory instance
*/
public static FSDirectory newFSDirectory(String clazzName, File file) {
public static FSDirectory newFSDirectory(String clazzName, Path path) {
try {
final Class<? extends FSDirectory> clazz = loadFSDirectoryClass(clazzName);
return newFSDirectory(clazz, file);
return newFSDirectory(clazz, path);
} catch (ClassNotFoundException e) {
throw new IllegalArgumentException(FSDirectory.class.getSimpleName()
+ " implementation not found: " + clazzName, e);
@ -52,7 +52,7 @@ public final class CommandLineUtil {
+ " implementation", e);
} catch (NoSuchMethodException e) {
throw new IllegalArgumentException(clazzName + " constructor with "
+ File.class.getSimpleName() + " as parameter not found", e);
+ Path.class.getSimpleName() + " as parameter not found", e);
} catch (Exception e) {
throw new IllegalArgumentException("Error creating " + clazzName + " instance", e);
}
@ -95,18 +95,18 @@ public final class CommandLineUtil {
/**
* Creates a new specific FSDirectory instance
* @param clazz The class of the object to be created
* @param file The file to be used as parameter constructor
* @param path The file to be used as parameter constructor
* @return The new FSDirectory instance
* @throws NoSuchMethodException If the Directory does not have a constructor that takes <code>File</code>.
* @throws NoSuchMethodException If the Directory does not have a constructor that takes <code>Path</code>.
* @throws InstantiationException If the class is abstract or an interface.
* @throws IllegalAccessException If the constructor does not have public visibility.
* @throws InvocationTargetException If the constructor throws an exception
*/
public static FSDirectory newFSDirectory(Class<? extends FSDirectory> clazz, File file)
public static FSDirectory newFSDirectory(Class<? extends FSDirectory> clazz, Path path)
throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException {
// Assuming every FSDirectory has a ctor(File):
Constructor<? extends FSDirectory> ctor = clazz.getConstructor(File.class);
return ctor.newInstance(file);
// Assuming every FSDirectory has a ctor(Path):
Constructor<? extends FSDirectory> ctor = clazz.getConstructor(Path.class);
return ctor.newInstance(path);
}
}

140
lucene/core/src/java/org/apache/lucene/util/IOUtils.java
View File

@ -21,9 +21,6 @@ import org.apache.lucene.store.Directory;
import java.io.BufferedReader;
import java.io.Closeable;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
@ -33,8 +30,12 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
import java.nio.charset.StandardCharsets;
import java.nio.file.FileVisitResult;
import java.nio.file.FileVisitor;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.nio.file.attribute.BasicFileAttributes;
import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Map;
@ -154,34 +155,6 @@ public final class IOUtils {
.onUnmappableCharacter(CodingErrorAction.REPORT);
return new BufferedReader(new InputStreamReader(stream, charSetDecoder));
}
/**
* Opens a Reader for the given {@link File} using a {@link CharsetDecoder}.
* Unlike Java's defaults this reader will throw an exception if your it detects
* the read charset doesn't match the expected {@link Charset}.
* <p>
* Decoding readers are useful to load configuration files, stopword lists or synonym files
* to detect character set problems. However, its not recommended to use as a common purpose
* reader.
* @param file the file to open a reader on
* @param charSet the expected charset
* @return a reader to read the given file
*/
public static Reader getDecodingReader(File file, Charset charSet) throws IOException {
FileInputStream stream = null;
boolean success = false;
try {
stream = new FileInputStream(file);
final Reader reader = getDecodingReader(stream, charSet);
success = true;
return reader;
} finally {
if (!success) {
IOUtils.close(stream);
}
}
}
/**
* Opens a Reader for the given resource using a {@link CharsetDecoder}.
@ -233,7 +206,7 @@ public final class IOUtils {
* <p>
* Some of the files may be null, if so they are ignored.
*/
public static void deleteFilesIgnoringExceptions(File... files) {
public static void deleteFilesIgnoringExceptions(Path... files) {
deleteFilesIgnoringExceptions(Arrays.asList(files));
}
@ -242,11 +215,11 @@ public final class IOUtils {
* <p>
* Some of the files may be null, if so they are ignored.
*/
public static void deleteFilesIgnoringExceptions(Iterable<? extends File> files) {
for (File name : files) {
public static void deleteFilesIgnoringExceptions(Iterable<? extends Path> files) {
for (Path name : files) {
if (name != null) {
try {
Files.delete(name.toPath());
Files.delete(name);
} catch (Throwable ignored) {
// ignore
}
@ -255,7 +228,7 @@ public final class IOUtils {
}
/**
* Deletes all given <tt>File</tt>s, if they exist. Some of the
* Deletes all given <tt>Path</tt>s, if they exist. Some of the
* <tt>File</tt>s may be null; they are
* ignored. After everything is deleted, the method either
* throws the first exception it hit while deleting, or
@ -263,12 +236,12 @@ public final class IOUtils {
*
* @param files files to delete
*/
public static void deleteFilesIfExist(File... files) throws IOException {
public static void deleteFilesIfExist(Path... files) throws IOException {
deleteFilesIfExist(Arrays.asList(files));
}
/**
* Deletes all given <tt>File</tt>s, if they exist. Some of the
* Deletes all given <tt>Path</tt>s, if they exist. Some of the
* <tt>File</tt>s may be null; they are
* ignored. After everything is deleted, the method either
* throws the first exception it hit while deleting, or
@ -276,13 +249,13 @@ public final class IOUtils {
*
* @param files files to delete
*/
public static void deleteFilesIfExist(Iterable<? extends File> files) throws IOException {
public static void deleteFilesIfExist(Iterable<? extends Path> files) throws IOException {
Throwable th = null;
for (File file : files) {
for (Path file : files) {
try {
if (file != null) {
Files.deleteIfExists(file.toPath());
Files.deleteIfExists(file);
}
} catch (Throwable t) {
addSuppressed(th, t);
@ -301,13 +274,13 @@ public final class IOUtils {
* @throws IOException if any of the given files (or their subhierarchy files in case
* of directories) cannot be removed.
*/
public static void rm(File... locations) throws IOException {
LinkedHashMap<File,Throwable> unremoved = rm(new LinkedHashMap<File,Throwable>(), locations);
public static void rm(Path... locations) throws IOException {
LinkedHashMap<Path,Throwable> unremoved = rm(new LinkedHashMap<Path,Throwable>(), locations);
if (!unremoved.isEmpty()) {
StringBuilder b = new StringBuilder("Could not remove the following files (in the order of attempts):\n");
for (Map.Entry<File,Throwable> kv : unremoved.entrySet()) {
for (Map.Entry<Path,Throwable> kv : unremoved.entrySet()) {
b.append(" ")
.append(kv.getKey().getAbsolutePath())
.append(kv.getKey().toAbsolutePath())
.append(": ")
.append(kv.getValue())
.append("\n");
@ -316,18 +289,50 @@ public final class IOUtils {
}
}
private static LinkedHashMap<File,Throwable> rm(LinkedHashMap<File,Throwable> unremoved, File... locations) {
private static LinkedHashMap<Path,Throwable> rm(final LinkedHashMap<Path,Throwable> unremoved, Path... locations) {
if (locations != null) {
for (File location : locations) {
if (location != null && location.exists()) {
if (location.isDirectory()) {
rm(unremoved, location.listFiles());
}
for (Path location : locations) {
// TODO: remove this leniency!
if (location != null && Files.exists(location)) {
try {
Files.delete(location.toPath());
} catch (Throwable cause) {
unremoved.put(location, cause);
Files.walkFileTree(location, new FileVisitor<Path>() {
@Override
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult postVisitDirectory(Path dir, IOException impossible) throws IOException {
assert impossible == null;
try {
Files.delete(dir);
} catch (IOException e) {
unremoved.put(dir, e);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {
try {
Files.delete(file);
} catch (IOException exc) {
unremoved.put(file, exc);
}
return FileVisitResult.CONTINUE;
}
@Override
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
if (exc != null) {
unremoved.put(file, exc);
}
return FileVisitResult.CONTINUE;
}
});
} catch (IOException impossible) {
throw new AssertionError("visitor threw exception", impossible);
}
}
}
@ -335,27 +340,6 @@ public final class IOUtils {
return unremoved;
}
/**
* Copy one file's contents to another file. The target will be overwritten
* if it exists. The source must exist.
*/
public static void copy(File source, File target) throws IOException {
FileInputStream fis = null;
FileOutputStream fos = null;
try {
fis = new FileInputStream(source);
fos = new FileOutputStream(target);
final byte [] buffer = new byte [1024 * 8];
int len;
while ((len = fis.read(buffer)) > 0) {
fos.write(buffer, 0, len);
}
} finally {
close(fis, fos);
}
}
/**
* Simple utilty method that takes a previously caught
* {@code Throwable} and rethrows either {@code
@ -394,12 +378,12 @@ public final class IOUtils {
* @param isDir if true, the given file is a directory (we open for read and ignore IOExceptions,
* because not all file systems and operating systems allow to fsync on a directory)
*/
public static void fsync(File fileToSync, boolean isDir) throws IOException {
public static void fsync(Path fileToSync, boolean isDir) throws IOException {
IOException exc = null;
// If the file is a directory we have to open read-only, for regular files we must open r/w for the fsync to have an effect.
// See http://blog.httrack.com/blog/2013/11/15/everything-you-always-wanted-to-know-about-fsync/
try (final FileChannel file = FileChannel.open(fileToSync.toPath(), isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) {
try (final FileChannel file = FileChannel.open(fileToSync, isDir ? StandardOpenOption.READ : StandardOpenOption.WRITE)) {
for (int retry = 0; retry < 5; retry++) {
try {
file.force(true);

76
lucene/core/src/java/org/apache/lucene/util/OfflineSorter.java
View File

@ -25,13 +25,11 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.DataOutputStream;
import java.io.EOFException;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.nio.file.StandardCopyOption;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
@ -45,7 +43,7 @@ import java.util.Locale;
* <li>exactly the above count of bytes for the sequence to be sorted.
* </ul>
*
* @see #sort(File, File)
* @see #sort(Path, Path)
* @lucene.experimental
* @lucene.internal
*/
@ -167,7 +165,7 @@ public final class OfflineSorter {
}
private final BufferSize ramBufferSize;
private final File tempDirectory;
private final Path tempDirectory;
private final Counter bufferBytesUsed = Counter.newCounter();
private final BytesRefArray buffer = new BytesRefArray(bufferBytesUsed);
@ -201,7 +199,7 @@ public final class OfflineSorter {
/**
* All-details constructor.
*/
public OfflineSorter(Comparator<BytesRef> comparator, BufferSize ramBufferSize, File tempDirectory, int maxTempfiles) {
public OfflineSorter(Comparator<BytesRef> comparator, BufferSize ramBufferSize, Path tempDirectory, int maxTempfiles) {
if (ramBufferSize.bytes < ABSOLUTE_MIN_SORT_BUFFER_SIZE) {
throw new IllegalArgumentException(MIN_BUFFER_SIZE_MSG + ": " + ramBufferSize.bytes);
}
@ -220,13 +218,13 @@ public final class OfflineSorter {
* Sort input to output, explicit hint for the buffer size. The amount of allocated
* memory may deviate from the hint (may be smaller or larger).
*/
public SortInfo sort(File input, File output) throws IOException {
public SortInfo sort(Path input, Path output) throws IOException {
sortInfo = new SortInfo();
sortInfo.totalTime = System.currentTimeMillis();
Files.deleteIfExists(output.toPath());
Files.deleteIfExists(output);
ArrayList<File> merges = new ArrayList<>();
ArrayList<Path> merges = new ArrayList<>();
boolean success3 = false;
try {
ByteSequencesReader is = new ByteSequencesReader(input);
@ -240,7 +238,7 @@ public final class OfflineSorter {
// Handle intermediate merges.
if (merges.size() == maxTempFiles) {
File intermediate = File.createTempFile("sort", "intermediate", tempDirectory);
Path intermediate = Files.createTempFile(tempDirectory, "sort", "intermediate");
boolean success2 = false;
try {
mergePartitions(merges, intermediate);
@ -267,11 +265,13 @@ public final class OfflineSorter {
// One partition, try to rename or copy if unsuccessful.
if (merges.size() == 1) {
File single = merges.get(0);
Path single = merges.get(0);
// If simple rename doesn't work this means the output is
// on a different volume or something. Copy the input then.
if (!single.renameTo(output)) {
copy(single, output);
try {
Files.move(single, output, StandardCopyOption.ATOMIC_MOVE);
} catch (IOException | UnsupportedOperationException e) {
Files.copy(single, output);
}
} else {
// otherwise merge the partitions with a priority queue.
@ -295,43 +295,23 @@ public final class OfflineSorter {
* Returns the default temporary directory. By default, java.io.tmpdir. If not accessible
* or not available, an IOException is thrown
*/
public static File defaultTempDir() throws IOException {
public static Path defaultTempDir() throws IOException {
String tempDirPath = System.getProperty("java.io.tmpdir");
if (tempDirPath == null)
throw new IOException("Java has no temporary folder property (java.io.tmpdir)?");
File tempDirectory = new File(tempDirPath);
if (!tempDirectory.exists() || !tempDirectory.canWrite()) {
Path tempDirectory = Paths.get(tempDirPath);
if (!Files.isWritable(tempDirectory)) {
throw new IOException("Java's temporary folder not present or writeable?: "
+ tempDirectory.getAbsolutePath());
+ tempDirectory.toAbsolutePath());
}
return tempDirectory;
}
/**
* Copies one file to another.
*/
private static void copy(File file, File output) throws IOException {
// 64kb copy buffer (empirical pick).
byte [] buffer = new byte [16 * 1024];
InputStream is = null;
OutputStream os = null;
try {
is = new FileInputStream(file);
os = new FileOutputStream(output);
int length;
while ((length = is.read(buffer)) > 0) {
os.write(buffer, 0, length);
}
} finally {
IOUtils.close(is, os);
}
}
/** Sort a single partition in-memory. */
protected File sortPartition(int len) throws IOException {
protected Path sortPartition(int len) throws IOException {
BytesRefArray data = this.buffer;
File tempFile = File.createTempFile("sort", "partition", tempDirectory);
Path tempFile = Files.createTempFile(tempDirectory, "sort", "partition");
long start = System.currentTimeMillis();
sortInfo.sortTime += (System.currentTimeMillis() - start);
@ -356,7 +336,7 @@ public final class OfflineSorter {
}
/** Merge a list of sorted temporary files (partitions) into an output file */
void mergePartitions(List<File> merges, File outputFile) throws IOException {
void mergePartitions(List<Path> merges, Path outputFile) throws IOException {
long start = System.currentTimeMillis();
ByteSequencesWriter out = new ByteSequencesWriter(outputFile);
@ -441,11 +421,11 @@ public final class OfflineSorter {
public static class ByteSequencesWriter implements Closeable {
private final DataOutput os;
/** Constructs a ByteSequencesWriter to the provided File */
public ByteSequencesWriter(File file) throws IOException {
/** Constructs a ByteSequencesWriter to the provided Path */
public ByteSequencesWriter(Path path) throws IOException {
this(new DataOutputStream(
new BufferedOutputStream(
new FileOutputStream(file))));
Files.newOutputStream(path))));
}
/** Constructs a ByteSequencesWriter to the provided DataOutput */
@ -505,11 +485,11 @@ public final class OfflineSorter {
public static class ByteSequencesReader implements Closeable {
private final DataInput is;
/** Constructs a ByteSequencesReader from the provided File */
public ByteSequencesReader(File file) throws IOException {
/** Constructs a ByteSequencesReader from the provided Path */
public ByteSequencesReader(Path path) throws IOException {
this(new DataInputStream(
new BufferedInputStream(
new FileInputStream(file))));
Files.newInputStream(path))));
}
/** Constructs a ByteSequencesReader from the provided DataInput */

36
lucene/core/src/java/org/apache/lucene/util/fst/FST.java
View File

@ -19,12 +19,11 @@ package org.apache.lucene.util.fst;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
@ -614,37 +613,18 @@ public final class FST<T> implements Accountable {
/**
* Writes an automaton to a file.
*/
public void save(final File file) throws IOException {
boolean success = false;
OutputStream os = new BufferedOutputStream(new FileOutputStream(file));
try {
save(new OutputStreamDataOutput(os));
success = true;
} finally {
if (success) {
IOUtils.close(os);
} else {
IOUtils.closeWhileHandlingException(os);
}
public void save(final Path path) throws IOException {
try (OutputStream os = Files.newOutputStream(path)) {
save(new OutputStreamDataOutput(new BufferedOutputStream(os)));
}
}
/**
* Reads an automaton from a file.
*/
public static <T> FST<T> read(File file, Outputs<T> outputs) throws IOException {
InputStream is = new BufferedInputStream(new FileInputStream(file));
boolean success = false;
try {
FST<T> fst = new FST<>(new InputStreamDataInput(is), outputs);
success = true;
return fst;
} finally {
if (success) {
IOUtils.close(is);
} else {
IOUtils.closeWhileHandlingException(is);
}
public static <T> FST<T> read(Path path, Outputs<T> outputs) throws IOException {
try (InputStream is = Files.newInputStream(path)) {
return new FST<>(new InputStreamDataInput(new BufferedInputStream(is)), outputs);
}
}

3
lucene/core/src/test/org/apache/lucene/index/TestAtomicUpdate.java
View File

@ -17,6 +17,7 @@ package org.apache.lucene.index;
*/
import java.io.File;
import java.nio.file.Path;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.*;
@ -174,7 +175,7 @@ public class TestAtomicUpdate extends LuceneTestCase {
directory.close();
// Second in an FSDirectory:
File dirPath = createTempDir("lucene.test.atomic");
Path dirPath = createTempDir("lucene.test.atomic");
directory = newFSDirectory(dirPath);
runTest(directory);
directory.close();

Some files were not shown because too many files have changed in this diff Show More