mirror of https://github.com/apache/lucene.git
LUCENE-6813: OfflineSorter no longer removes its output file up front; fix file handle leak in RangeTreeWriter
git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1705155 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
parent
fb881a4f0f
commit
4fa7bac977
|
@ -154,6 +154,18 @@ Bug Fixes
|
|||
spatial module. See https://github.com/locationtech/spatial4j/blob/master/CHANGES.md
|
||||
(David Smiley)
|
||||
|
||||
* LUCENE-6813: OfflineSorter no longer removes its output Path up
|
||||
front, and instead opens it for write with the
|
||||
StandardCopyOption.REPLACE_EXISTING to overwrite any prior file, so
|
||||
that callers can safely use Files.createTempFile for the output.
|
||||
This change also fixes OfflineSorter's default temp directory when
|
||||
running tests to use mock filesystems so e.g. we detect file handle
|
||||
leaks (Dawid Weiss, Robert Muir, Mike McCandless)
|
||||
|
||||
* LUCENE-6813: RangeTreeWriter was failing to close all file handles
|
||||
it opened, leading to intermittent failures on Windows (Dawid Weiss,
|
||||
Robert Muir, Mike McCandless)
|
||||
|
||||
Other
|
||||
|
||||
* LUCENE-6812: Upgrade RandomizedTesting to 2.1.17. (Dawid Weiss)
|
||||
|
|
|
@ -139,7 +139,7 @@ public class Dictionary {
|
|||
// when set, some words have exceptional stems, and the last entry is a pointer to stemExceptions
|
||||
boolean hasStemExceptions;
|
||||
|
||||
private final Path tempDir = OfflineSorter.defaultTempDir(); // TODO: make this configurable?
|
||||
private final Path tempDir = OfflineSorter.getDefaultTempDir(); // TODO: make this configurable?
|
||||
|
||||
boolean ignoreCase;
|
||||
boolean complexPrefixes;
|
||||
|
|
|
@ -48,6 +48,9 @@ import java.util.Locale;
|
|||
* @lucene.internal
|
||||
*/
|
||||
public final class OfflineSorter {
|
||||
|
||||
private static Path DEFAULT_TEMP_DIR;
|
||||
|
||||
/** Convenience constant for megabytes */
|
||||
public final static long MB = 1024 * 1024;
|
||||
/** Convenience constant for gigabytes */
|
||||
|
@ -179,21 +182,21 @@ public final class OfflineSorter {
|
|||
/**
|
||||
* Defaults constructor.
|
||||
*
|
||||
* @see #defaultTempDir()
|
||||
* @see #getDefaultTempDir()
|
||||
* @see BufferSize#automatic()
|
||||
*/
|
||||
public OfflineSorter() throws IOException {
|
||||
this(DEFAULT_COMPARATOR, BufferSize.automatic(), defaultTempDir(), MAX_TEMPFILES);
|
||||
this(DEFAULT_COMPARATOR, BufferSize.automatic(), getDefaultTempDir(), MAX_TEMPFILES);
|
||||
}
|
||||
|
||||
/**
|
||||
* Defaults constructor with a custom comparator.
|
||||
*
|
||||
* @see #defaultTempDir()
|
||||
* @see #getDefaultTempDir()
|
||||
* @see BufferSize#automatic()
|
||||
*/
|
||||
public OfflineSorter(Comparator<BytesRef> comparator) throws IOException {
|
||||
this(comparator, BufferSize.automatic(), defaultTempDir(), MAX_TEMPFILES);
|
||||
this(comparator, BufferSize.automatic(), getDefaultTempDir(), MAX_TEMPFILES);
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -222,7 +225,9 @@ public final class OfflineSorter {
|
|||
sortInfo = new SortInfo();
|
||||
sortInfo.totalTime = System.currentTimeMillis();
|
||||
|
||||
Files.deleteIfExists(output);
|
||||
// NOTE: don't remove output here: its existence (often created by the caller
|
||||
// up above using Files.createTempFile) prevents another concurrent caller
|
||||
// of this API (from a different thread) from incorrectly re-using this file name
|
||||
|
||||
ArrayList<Path> merges = new ArrayList<>();
|
||||
boolean success3 = false;
|
||||
|
@ -257,22 +262,16 @@ public final class OfflineSorter {
|
|||
}
|
||||
success = true;
|
||||
} finally {
|
||||
if (success)
|
||||
if (success) {
|
||||
IOUtils.close(is);
|
||||
else
|
||||
} else {
|
||||
IOUtils.closeWhileHandlingException(is);
|
||||
}
|
||||
}
|
||||
|
||||
// One partition, try to rename or copy if unsuccessful.
|
||||
if (merges.size() == 1) {
|
||||
Path single = merges.get(0);
|
||||
// If simple rename doesn't work this means the output is
|
||||
// on a different volume or something. Copy the input then.
|
||||
try {
|
||||
Files.move(single, output, StandardCopyOption.ATOMIC_MOVE);
|
||||
} catch (IOException | UnsupportedOperationException e) {
|
||||
Files.copy(single, output);
|
||||
}
|
||||
Files.move(merges.get(0), output, StandardCopyOption.REPLACE_EXISTING);
|
||||
} else {
|
||||
// otherwise merge the partitions with a priority queue.
|
||||
mergePartitions(merges, output);
|
||||
|
@ -291,21 +290,31 @@ public final class OfflineSorter {
|
|||
return sortInfo;
|
||||
}
|
||||
|
||||
/** Used by test framework */
|
||||
static void setDefaultTempDir(Path tempDir) {
|
||||
DEFAULT_TEMP_DIR = tempDir;
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns the default temporary directory. By default, java.io.tmpdir. If not accessible
|
||||
* or not available, an IOException is thrown
|
||||
*/
|
||||
public static Path defaultTempDir() throws IOException {
|
||||
String tempDirPath = System.getProperty("java.io.tmpdir");
|
||||
if (tempDirPath == null)
|
||||
throw new IOException("Java has no temporary folder property (java.io.tmpdir)?");
|
||||
|
||||
Path tempDirectory = Paths.get(tempDirPath);
|
||||
if (!Files.isWritable(tempDirectory)) {
|
||||
throw new IOException("Java's temporary folder not present or writeable?: "
|
||||
+ tempDirectory.toAbsolutePath());
|
||||
public synchronized static Path getDefaultTempDir() throws IOException {
|
||||
if (DEFAULT_TEMP_DIR == null) {
|
||||
// Lazy init
|
||||
String tempDirPath = System.getProperty("java.io.tmpdir");
|
||||
if (tempDirPath == null) {
|
||||
throw new IOException("Java has no temporary folder property (java.io.tmpdir)?");
|
||||
}
|
||||
Path tempDirectory = Paths.get(tempDirPath);
|
||||
if (Files.isWritable(tempDirectory) == false) {
|
||||
throw new IOException("Java's temporary folder not present or writeable?: "
|
||||
+ tempDirectory.toAbsolutePath());
|
||||
}
|
||||
DEFAULT_TEMP_DIR = tempDirectory;
|
||||
}
|
||||
return tempDirectory;
|
||||
|
||||
return DEFAULT_TEMP_DIR;
|
||||
}
|
||||
|
||||
/** Sort a single partition in-memory. */
|
||||
|
|
|
@ -25,13 +25,14 @@ import java.nio.file.Path;
|
|||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Comparator;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.LuceneTestCase;
|
||||
import org.apache.lucene.util.OfflineSorter;
|
||||
import org.apache.lucene.util.OfflineSorter.BufferSize;
|
||||
import org.apache.lucene.util.OfflineSorter.ByteSequencesWriter;
|
||||
import org.apache.lucene.util.OfflineSorter.SortInfo;
|
||||
import org.apache.lucene.util.OfflineSorter;
|
||||
|
||||
/**
|
||||
* Tests for on-disk merge sorting.
|
||||
|
@ -47,8 +48,9 @@ public class TestOfflineSorter extends LuceneTestCase {
|
|||
|
||||
@Override
|
||||
public void tearDown() throws Exception {
|
||||
if (tempDir != null)
|
||||
if (tempDir != null) {
|
||||
IOUtils.rm(tempDir);
|
||||
}
|
||||
super.tearDown();
|
||||
}
|
||||
|
||||
|
@ -64,14 +66,14 @@ public class TestOfflineSorter extends LuceneTestCase {
|
|||
|
||||
public void testIntermediateMerges() throws Exception {
|
||||
// Sort 20 mb worth of data with 1mb buffer, binary merging.
|
||||
SortInfo info = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.defaultTempDir(), 2),
|
||||
SortInfo info = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.getDefaultTempDir(), 2),
|
||||
generateRandom((int)OfflineSorter.MB * 20));
|
||||
assertTrue(info.mergeRounds > 10);
|
||||
}
|
||||
|
||||
public void testSmallRandom() throws Exception {
|
||||
// Sort 20 mb worth of data with 1mb buffer.
|
||||
SortInfo sortInfo = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.defaultTempDir(), OfflineSorter.MAX_TEMPFILES),
|
||||
SortInfo sortInfo = checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), OfflineSorter.getDefaultTempDir(), OfflineSorter.MAX_TEMPFILES),
|
||||
generateRandom((int)OfflineSorter.MB * 20));
|
||||
assertEquals(1, sortInfo.mergeRounds);
|
||||
}
|
||||
|
@ -79,17 +81,17 @@ public class TestOfflineSorter extends LuceneTestCase {
|
|||
@Nightly
|
||||
public void testLargerRandom() throws Exception {
|
||||
// Sort 100MB worth of data with 15mb buffer.
|
||||
checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(16), OfflineSorter.defaultTempDir(), OfflineSorter.MAX_TEMPFILES),
|
||||
checkSort(new OfflineSorter(OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(16), OfflineSorter.getDefaultTempDir(), OfflineSorter.MAX_TEMPFILES),
|
||||
generateRandom((int)OfflineSorter.MB * 100));
|
||||
}
|
||||
|
||||
private byte[][] generateRandom(int howMuchData) {
|
||||
private byte[][] generateRandom(int howMuchDataInBytes) {
|
||||
ArrayList<byte[]> data = new ArrayList<>();
|
||||
while (howMuchData > 0) {
|
||||
byte [] current = new byte [random().nextInt(256)];
|
||||
while (howMuchDataInBytes > 0) {
|
||||
byte[] current = new byte[random().nextInt(256)];
|
||||
random().nextBytes(current);
|
||||
data.add(current);
|
||||
howMuchData -= current.length;
|
||||
howMuchDataInBytes -= current.length;
|
||||
}
|
||||
byte [][] bytes = data.toArray(new byte[data.size()][]);
|
||||
return bytes;
|
||||
|
@ -107,6 +109,7 @@ public class TestOfflineSorter extends LuceneTestCase {
|
|||
return left.length - right.length;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Check sorting data on an instance of {@link OfflineSorter}.
|
||||
*/
|
||||
|
@ -116,12 +119,17 @@ public class TestOfflineSorter extends LuceneTestCase {
|
|||
Arrays.sort(data, unsignedByteOrderComparator);
|
||||
Path golden = writeAll("golden", data);
|
||||
|
||||
Path sorted = tempDir.resolve("sorted");
|
||||
SortInfo sortInfo = sort.sort(unsorted, sorted);
|
||||
//System.out.println("Input size [MB]: " + unsorted.length() / (1024 * 1024));
|
||||
//System.out.println(sortInfo);
|
||||
Path sorted = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "sorted", "");
|
||||
SortInfo sortInfo;
|
||||
try {
|
||||
sortInfo = sort.sort(unsorted, sorted);
|
||||
//System.out.println("Input size [MB]: " + unsorted.length() / (1024 * 1024));
|
||||
//System.out.println(sortInfo);
|
||||
assertFilesIdentical(golden, sorted);
|
||||
} finally {
|
||||
IOUtils.rm(unsorted, golden, sorted);
|
||||
}
|
||||
|
||||
assertFilesIdentical(golden, sorted);
|
||||
return sortInfo;
|
||||
}
|
||||
|
||||
|
@ -146,7 +154,7 @@ public class TestOfflineSorter extends LuceneTestCase {
|
|||
}
|
||||
|
||||
private Path writeAll(String name, byte[][] data) throws IOException {
|
||||
Path file = tempDir.resolve(name);
|
||||
Path file = Files.createTempFile(tempDir, name, "");
|
||||
ByteSequencesWriter w = new OfflineSorter.ByteSequencesWriter(file);
|
||||
for (byte [] datum : data) {
|
||||
w.write(datum);
|
||||
|
@ -181,4 +189,32 @@ public class TestOfflineSorter extends LuceneTestCase {
|
|||
} catch (IllegalArgumentException e) {
|
||||
}
|
||||
}
|
||||
|
||||
public void testThreadSafety() throws Exception {
|
||||
Thread[] threads = new Thread[TestUtil.nextInt(random(), 4, 10)];
|
||||
final AtomicBoolean failed = new AtomicBoolean();
|
||||
final int iters = atLeast(1000);
|
||||
for(int i=0;i<threads.length;i++) {
|
||||
threads[i] = new Thread() {
|
||||
@Override
|
||||
public void run() {
|
||||
try {
|
||||
for(int iter=0;iter<iters && failed.get() == false;iter++) {
|
||||
checkSort(new OfflineSorter(), generateRandom(1024));
|
||||
}
|
||||
} catch (Throwable th) {
|
||||
failed.set(true);
|
||||
throw new RuntimeException(th);
|
||||
}
|
||||
}
|
||||
};
|
||||
threads[i].start();
|
||||
}
|
||||
|
||||
for(Thread thread : threads) {
|
||||
thread.join();
|
||||
}
|
||||
|
||||
assertFalse(failed.get());
|
||||
}
|
||||
}
|
||||
|
|
|
@ -92,7 +92,6 @@ class BKDTreeWriter {
|
|||
private GrowingHeapLatLonWriter heapWriter;
|
||||
|
||||
private Path tempInput;
|
||||
private Path tempDir;
|
||||
private final int maxPointsInLeafNode;
|
||||
private final int maxPointsSortInHeap;
|
||||
|
||||
|
@ -143,11 +142,8 @@ class BKDTreeWriter {
|
|||
/** If the current segment has too many points then we switchover to temp files / offline sort. */
|
||||
private void switchToOffline() throws IOException {
|
||||
|
||||
// OfflineSorter isn't thread safe, but our own private tempDir works around this:
|
||||
tempDir = Files.createTempDirectory(OfflineSorter.defaultTempDir(), BKDTreeWriter.class.getSimpleName());
|
||||
|
||||
// For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree:
|
||||
tempInput = tempDir.resolve("in");
|
||||
tempInput = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "in", "");
|
||||
writer = new OfflineSorter.ByteSequencesWriter(tempInput);
|
||||
for(int i=0;i<pointCount;i++) {
|
||||
scratchBytesOutput.reset(scratchBytes);
|
||||
|
@ -293,7 +289,7 @@ class BKDTreeWriter {
|
|||
} else {
|
||||
|
||||
// Offline sort:
|
||||
assert tempDir != null;
|
||||
assert tempInput != null;
|
||||
|
||||
final ByteArrayDataInput reader = new ByteArrayDataInput();
|
||||
Comparator<BytesRef> cmp = new Comparator<BytesRef>() {
|
||||
|
@ -333,10 +329,11 @@ class BKDTreeWriter {
|
|||
}
|
||||
};
|
||||
|
||||
Path sorted = tempDir.resolve("sorted");
|
||||
Path sorted = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "sorted", "");
|
||||
boolean success = false;
|
||||
|
||||
try {
|
||||
OfflineSorter latSorter = new OfflineSorter(cmp, OfflineSorter.BufferSize.automatic(), tempDir, OfflineSorter.MAX_TEMPFILES);
|
||||
OfflineSorter latSorter = new OfflineSorter(cmp);
|
||||
latSorter.sort(tempInput, sorted);
|
||||
LatLonWriter writer = convertToFixedWidth(sorted);
|
||||
success = true;
|
||||
|
@ -443,12 +440,6 @@ class BKDTreeWriter {
|
|||
out.writeVLong(leafBlockFPs[i]);
|
||||
}
|
||||
|
||||
if (tempDir != null) {
|
||||
// If we had to go offline, we should have removed all temp files we wrote:
|
||||
assert directoryIsEmpty(tempDir);
|
||||
IOUtils.rm(tempDir);
|
||||
}
|
||||
|
||||
return indexFP;
|
||||
}
|
||||
|
||||
|
@ -835,7 +826,7 @@ class BKDTreeWriter {
|
|||
if (count < maxPointsSortInHeap) {
|
||||
return new HeapLatLonWriter((int) count);
|
||||
} else {
|
||||
return new OfflineLatLonWriter(tempDir, count);
|
||||
return new OfflineLatLonWriter(count);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -17,15 +17,16 @@ package org.apache.lucene.bkdtree;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.store.OutputStreamDataOutput;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.OfflineSorter;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.store.OutputStreamDataOutput;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
||||
final class OfflineLatLonWriter implements LatLonWriter {
|
||||
|
||||
final Path tempFile;
|
||||
|
@ -36,8 +37,8 @@ final class OfflineLatLonWriter implements LatLonWriter {
|
|||
private long countWritten;
|
||||
private boolean closed;
|
||||
|
||||
public OfflineLatLonWriter(Path tempDir, long count) throws IOException {
|
||||
tempFile = Files.createTempFile(tempDir, "size" + count + ".", "");
|
||||
public OfflineLatLonWriter(long count) throws IOException {
|
||||
tempFile = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "size" + count + ".", "");
|
||||
out = new OutputStreamDataOutput(new BufferedOutputStream(Files.newOutputStream(tempFile)));
|
||||
this.count = count;
|
||||
}
|
||||
|
|
|
@ -17,15 +17,16 @@ package org.apache.lucene.rangetree;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.store.OutputStreamDataOutput;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.OfflineSorter;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.store.OutputStreamDataOutput;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
||||
final class OfflineSliceWriter implements SliceWriter {
|
||||
|
||||
final Path tempFile;
|
||||
|
@ -36,8 +37,8 @@ final class OfflineSliceWriter implements SliceWriter {
|
|||
private boolean closed;
|
||||
private long countWritten;
|
||||
|
||||
public OfflineSliceWriter(Path tempDir, long count) throws IOException {
|
||||
tempFile = Files.createTempFile(tempDir, "size" + count + ".", "");
|
||||
public OfflineSliceWriter(long count) throws IOException {
|
||||
tempFile = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "size" + count + ".", "");
|
||||
out = new OutputStreamDataOutput(new BufferedOutputStream(Files.newOutputStream(tempFile)));
|
||||
this.count = count;
|
||||
}
|
||||
|
|
|
@ -81,7 +81,6 @@ class RangeTreeWriter {
|
|||
private GrowingHeapSliceWriter heapWriter;
|
||||
|
||||
private Path tempInput;
|
||||
private Path tempDir;
|
||||
private final int maxValuesInLeafNode;
|
||||
private final int maxValuesSortInHeap;
|
||||
|
||||
|
@ -121,11 +120,8 @@ class RangeTreeWriter {
|
|||
/** If the current segment has too many points then we switchover to temp files / offline sort. */
|
||||
private void switchToOffline() throws IOException {
|
||||
|
||||
// OfflineSorter isn't thread safe, but our own private tempDir works around this:
|
||||
tempDir = Files.createTempDirectory(OfflineSorter.defaultTempDir(), RangeTreeWriter.class.getSimpleName());
|
||||
|
||||
// For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree:
|
||||
tempInput = tempDir.resolve("in");
|
||||
tempInput = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "in", "");
|
||||
writer = new OfflineSorter.ByteSequencesWriter(tempInput);
|
||||
for(int i=0;i<valueCount;i++) {
|
||||
scratchBytesOutput.reset(scratchBytes);
|
||||
|
@ -251,7 +247,7 @@ class RangeTreeWriter {
|
|||
} else {
|
||||
|
||||
// Offline sort:
|
||||
assert tempDir != null;
|
||||
assert tempInput != null;
|
||||
|
||||
final ByteArrayDataInput reader = new ByteArrayDataInput();
|
||||
Comparator<BytesRef> cmp = new Comparator<BytesRef>() {
|
||||
|
@ -284,10 +280,10 @@ class RangeTreeWriter {
|
|||
}
|
||||
};
|
||||
|
||||
Path sorted = tempDir.resolve("sorted");
|
||||
Path sorted = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "sorted", "");
|
||||
boolean success = false;
|
||||
try {
|
||||
OfflineSorter sorter = new OfflineSorter(cmp, OfflineSorter.BufferSize.automatic(), tempDir, OfflineSorter.MAX_TEMPFILES);
|
||||
OfflineSorter sorter = new OfflineSorter(cmp);
|
||||
sorter.sort(tempInput, sorted);
|
||||
SliceWriter writer = convertToFixedWidth(sorted);
|
||||
success = true;
|
||||
|
@ -387,12 +383,6 @@ class RangeTreeWriter {
|
|||
}
|
||||
out.writeLong(globalMaxValue);
|
||||
|
||||
if (tempDir != null) {
|
||||
// If we had to go offline, we should have removed all temp files we wrote:
|
||||
assert directoryIsEmpty(tempDir);
|
||||
IOUtils.rm(tempDir);
|
||||
}
|
||||
|
||||
return indexFP;
|
||||
}
|
||||
|
||||
|
@ -464,12 +454,15 @@ class RangeTreeWriter {
|
|||
// Cutover to heap:
|
||||
SliceWriter writer = new HeapSliceWriter((int) count);
|
||||
SliceReader reader = source.writer.getReader(source.start);
|
||||
for(int i=0;i<count;i++) {
|
||||
boolean hasNext = reader.next();
|
||||
assert hasNext;
|
||||
writer.append(reader.value(), reader.ord(), reader.docID());
|
||||
try {
|
||||
for(int i=0;i<count;i++) {
|
||||
boolean hasNext = reader.next();
|
||||
assert hasNext;
|
||||
writer.append(reader.value(), reader.ord(), reader.docID());
|
||||
}
|
||||
} finally {
|
||||
IOUtils.close(reader, writer);
|
||||
}
|
||||
writer.close();
|
||||
source = new PathSlice(writer, 0, count);
|
||||
}
|
||||
|
||||
|
@ -587,7 +580,7 @@ class RangeTreeWriter {
|
|||
if (count < maxValuesSortInHeap) {
|
||||
return new HeapSliceWriter((int) count);
|
||||
} else {
|
||||
return new OfflineSliceWriter(tempDir, count);
|
||||
return new OfflineSliceWriter(count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -55,8 +55,6 @@ import java.util.Set;
|
|||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
// TODO: can test framework assert we don't leak temp files?
|
||||
|
||||
public class TestBKDTree extends LuceneTestCase {
|
||||
|
||||
private static boolean smallBBox;
|
||||
|
|
|
@ -57,8 +57,6 @@ import java.util.Set;
|
|||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
|
||||
// TODO: can test framework assert we don't leak temp files?
|
||||
|
||||
public class TestRangeTree extends LuceneTestCase {
|
||||
|
||||
// Controls what range of values we randomly generate, so we sometimes test narrow ranges:
|
||||
|
|
|
@ -87,7 +87,6 @@ class BKD3DTreeWriter {
|
|||
private GrowingHeapWriter heapWriter;
|
||||
|
||||
private Path tempInput;
|
||||
private Path tempDir;
|
||||
private final int maxPointsInLeafNode;
|
||||
private final int maxPointsSortInHeap;
|
||||
|
||||
|
@ -128,11 +127,8 @@ class BKD3DTreeWriter {
|
|||
/** If the current segment has too many points then we switchover to temp files / offline sort. */
|
||||
private void switchToOffline() throws IOException {
|
||||
|
||||
// OfflineSorter isn't thread safe, but our own private tempDir works around this:
|
||||
tempDir = Files.createTempDirectory(OfflineSorter.defaultTempDir(), BKD3DTreeWriter.class.getSimpleName());
|
||||
|
||||
// For each .add we just append to this input file, then in .finish we sort this input and resursively build the tree:
|
||||
tempInput = tempDir.resolve("in");
|
||||
tempInput = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "in", "");
|
||||
writer = new OfflineSorter.ByteSequencesWriter(tempInput);
|
||||
for(int i=0;i<pointCount;i++) {
|
||||
scratchBytesOutput.reset(scratchBytes);
|
||||
|
@ -288,7 +284,7 @@ class BKD3DTreeWriter {
|
|||
} else {
|
||||
|
||||
// Offline sort:
|
||||
assert tempDir != null;
|
||||
assert tempInput != null;
|
||||
|
||||
final ByteArrayDataInput reader = new ByteArrayDataInput();
|
||||
Comparator<BytesRef> cmp = new Comparator<BytesRef>() {
|
||||
|
@ -332,10 +328,10 @@ class BKD3DTreeWriter {
|
|||
}
|
||||
};
|
||||
|
||||
Path sorted = tempDir.resolve("sorted");
|
||||
Path sorted = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "sorted", "");
|
||||
boolean success = false;
|
||||
try {
|
||||
OfflineSorter sorter = new OfflineSorter(cmp, OfflineSorter.BufferSize.automatic(), tempDir, OfflineSorter.MAX_TEMPFILES);
|
||||
OfflineSorter sorter = new OfflineSorter(cmp);
|
||||
sorter.sort(tempInput, sorted);
|
||||
Writer writer = convertToFixedWidth(sorted);
|
||||
success = true;
|
||||
|
@ -453,12 +449,6 @@ class BKD3DTreeWriter {
|
|||
out.writeVLong(leafBlockFPs[i]);
|
||||
}
|
||||
|
||||
if (tempDir != null) {
|
||||
// If we had to go offline, we should have removed all temp files we wrote:
|
||||
assert directoryIsEmpty(tempDir);
|
||||
IOUtils.rm(tempDir);
|
||||
}
|
||||
|
||||
return indexFP;
|
||||
}
|
||||
|
||||
|
@ -934,7 +924,7 @@ class BKD3DTreeWriter {
|
|||
if (count < maxPointsSortInHeap) {
|
||||
return new HeapWriter((int) count);
|
||||
} else {
|
||||
return new OfflineWriter(tempDir, count);
|
||||
return new OfflineWriter(count);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
@ -17,15 +17,16 @@ package org.apache.lucene.bkdtree3d;
|
|||
* limitations under the License.
|
||||
*/
|
||||
|
||||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.store.OutputStreamDataOutput;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
import org.apache.lucene.util.OfflineSorter;
|
||||
|
||||
import java.io.BufferedOutputStream;
|
||||
import java.io.IOException;
|
||||
import java.nio.file.Files;
|
||||
import java.nio.file.Path;
|
||||
|
||||
import org.apache.lucene.store.ByteArrayDataOutput;
|
||||
import org.apache.lucene.store.OutputStreamDataOutput;
|
||||
import org.apache.lucene.util.IOUtils;
|
||||
|
||||
final class OfflineWriter implements Writer {
|
||||
|
||||
final Path tempFile;
|
||||
|
@ -36,8 +37,8 @@ final class OfflineWriter implements Writer {
|
|||
private long countWritten;
|
||||
private boolean closed;
|
||||
|
||||
public OfflineWriter(Path tempDir, long count) throws IOException {
|
||||
tempFile = Files.createTempFile(tempDir, "size" + count + ".", "");
|
||||
public OfflineWriter(long count) throws IOException {
|
||||
tempFile = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "size" + count + ".", "");
|
||||
out = new OutputStreamDataOutput(new BufferedOutputStream(Files.newOutputStream(tempFile)));
|
||||
this.count = count;
|
||||
}
|
||||
|
|
|
@ -169,7 +169,7 @@ public class SortedInputIterator implements InputIterator {
|
|||
|
||||
private ByteSequencesReader sort() throws IOException {
|
||||
String prefix = getClass().getSimpleName();
|
||||
Path directory = OfflineSorter.defaultTempDir();
|
||||
Path directory = OfflineSorter.getDefaultTempDir();
|
||||
tempInput = Files.createTempFile(directory, prefix, ".input");
|
||||
tempSorted = Files.createTempFile(directory, prefix, ".sorted");
|
||||
|
||||
|
|
|
@ -397,7 +397,7 @@ public class AnalyzingSuggester extends Lookup implements Accountable {
|
|||
throw new IllegalArgumentException("this suggester doesn't support contexts");
|
||||
}
|
||||
String prefix = getClass().getSimpleName();
|
||||
Path directory = OfflineSorter.defaultTempDir();
|
||||
Path directory = OfflineSorter.getDefaultTempDir();
|
||||
Path tempInput = Files.createTempFile(directory, prefix, ".input");
|
||||
Path tempSorted = Files.createTempFile(directory, prefix, ".sorted");
|
||||
|
||||
|
|
|
@ -44,7 +44,7 @@ public class ExternalRefSorter implements BytesRefSorter, Closeable {
|
|||
*/
|
||||
public ExternalRefSorter(OfflineSorter sort) throws IOException {
|
||||
this.sort = sort;
|
||||
this.input = Files.createTempFile(OfflineSorter.defaultTempDir(), "RefSorter-", ".raw");
|
||||
this.input = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "RefSorter-", ".raw");
|
||||
this.writer = new OfflineSorter.ByteSequencesWriter(input);
|
||||
}
|
||||
|
||||
|
@ -59,7 +59,7 @@ public class ExternalRefSorter implements BytesRefSorter, Closeable {
|
|||
if (sorted == null) {
|
||||
closeWriter();
|
||||
|
||||
sorted = Files.createTempFile(OfflineSorter.defaultTempDir(), "RefSorter-", ".sorted");
|
||||
sorted = Files.createTempFile(OfflineSorter.getDefaultTempDir(), "RefSorter-", ".sorted");
|
||||
boolean success = false;
|
||||
try {
|
||||
sort.sort(input, sorted);
|
||||
|
|
|
@ -160,9 +160,9 @@ public class FSTCompletionLookup extends Lookup implements Accountable {
|
|||
throw new IllegalArgumentException("this suggester doesn't support contexts");
|
||||
}
|
||||
Path tempInput = Files.createTempFile(
|
||||
OfflineSorter.defaultTempDir(), FSTCompletionLookup.class.getSimpleName(), ".input");
|
||||
OfflineSorter.getDefaultTempDir(), FSTCompletionLookup.class.getSimpleName(), ".input");
|
||||
Path tempSorted = Files.createTempFile(
|
||||
OfflineSorter.defaultTempDir(), FSTCompletionLookup.class.getSimpleName(), ".sorted");
|
||||
OfflineSorter.getDefaultTempDir(), FSTCompletionLookup.class.getSimpleName(), ".sorted");
|
||||
|
||||
OfflineSorter.ByteSequencesWriter writer = new OfflineSorter.ByteSequencesWriter(tempInput);
|
||||
OfflineSorter.ByteSequencesReader reader = null;
|
||||
|
|
|
@ -61,7 +61,7 @@ public class AssertingSpanQuery extends SpanQuery {
|
|||
|
||||
@Override
|
||||
public Query clone() {
|
||||
return new AssertingSpanQuery((SpanQuery) in);
|
||||
return new AssertingSpanQuery(in);
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
|
@ -113,6 +113,9 @@ final class TestRuleTemporaryFilesCleanup extends TestRuleAdapter {
|
|||
assert tempDirBase == null;
|
||||
fileSystem = initializeFileSystem();
|
||||
javaTempDir = initializeJavaTempDir();
|
||||
|
||||
// So all code using OfflineSorter (suggesters, BKD tree, NumericRangeTree) see MockFS goodness, e.g. catching leaked file handles:
|
||||
OfflineSorter.setDefaultTempDir(javaTempDir);
|
||||
}
|
||||
|
||||
// os/config-independent limit for too many open files
|
||||
|
|
Loading…
Reference in New Issue