diff --git a/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java b/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java index 56a18c73fd6..af0cde305c8 100644 --- a/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java +++ b/lucene/src/test/org/apache/lucene/index/TestNRTThreads.java @@ -65,7 +65,7 @@ public class TestNRTThreads extends LuceneTestCase { CodecProvider.getDefault().setDefaultFieldCodec("Standard"); } - final LineFileDocs docs = new LineFileDocs(true); + final LineFileDocs docs = new LineFileDocs(random); final File tempDir = _TestUtil.getTempDir("nrtopenfiles"); final MockDirectoryWrapper dir = new MockDirectoryWrapper(random, FSDirectory.open(tempDir)); final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()); diff --git a/lucene/src/test/org/apache/lucene/util/LineFileDocs.java b/lucene/src/test/org/apache/lucene/util/LineFileDocs.java index fe3c0eb355a..56cb3e089d3 100644 --- a/lucene/src/test/org/apache/lucene/util/LineFileDocs.java +++ b/lucene/src/test/org/apache/lucene/util/LineFileDocs.java @@ -18,6 +18,7 @@ package org.apache.lucene.util; */ import java.io.Closeable; +import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.BufferedReader; @@ -26,6 +27,7 @@ import java.io.InputStream; import java.io.BufferedInputStream; import java.util.concurrent.atomic.AtomicInteger; import java.util.zip.GZIPInputStream; +import java.util.Random; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; @@ -36,21 +38,19 @@ import org.apache.lucene.document.Field; public class LineFileDocs implements Closeable { private BufferedReader reader; - private final boolean forever; private final static int BUFFER_SIZE = 1 << 16; // 64K private final AtomicInteger id = new AtomicInteger(); private final String path; // If forever is true, we rewind the file at EOF (repeat // the docs over and over) - public LineFileDocs(String path, boolean forever) throws IOException { + public LineFileDocs(Random random, String path) throws IOException { this.path = path; - this.forever = forever; - open(); + open(random); } - public LineFileDocs(boolean forever) throws IOException { - this(LuceneTestCase.TEST_LINE_DOCS_FILE, forever); + public LineFileDocs(Random random) throws IOException { + this(random, LuceneTestCase.TEST_LINE_DOCS_FILE); } public synchronized void close() throws IOException { @@ -60,22 +60,49 @@ public class LineFileDocs implements Closeable { } } - private synchronized void open() throws IOException { + private synchronized void open(Random random) throws IOException { InputStream is = getClass().getResourceAsStream(path); if (is == null) { // if its not in classpath, we load it as absolute filesystem path (e.g. Hudson's home dir) is = new FileInputStream(path); } + File file = new File(path); + long size; + if (file.exists()) { + size = file.length(); + } else { + size = is.available(); + } if (path.endsWith(".gz")) { is = new GZIPInputStream(is); + // guestimate: + size *= 2.8; } + final InputStream in = new BufferedInputStream(is, BUFFER_SIZE); reader = new BufferedReader(new InputStreamReader(in, "UTF-8"), BUFFER_SIZE); + + // Override sizes for currently "known" line files: + if (path.equals("europarl.lines.txt.gz")) { + size = 15129506L; + } else if (path.equals("/home/hudson/lucene-data/enwiki.random.lines.txt.gz")) { + size = 3038178822L; + } + + // Randomly seek to starting point: + if (random != null && size > 3) { + final long seekTo = (random.nextLong()&Long.MAX_VALUE) % (size/3); + if (LuceneTestCase.VERBOSE) { + System.out.println("TEST: LineFileDocs: seek to fp=" + seekTo + " on open"); + } + reader.skip(seekTo); + reader.readLine(); + } } - public synchronized void reset() throws IOException { + public synchronized void reset(Random random) throws IOException { close(); - open(); + open(random); id.set(0); } @@ -117,15 +144,13 @@ public class LineFileDocs implements Closeable { synchronized(this) { line = reader.readLine(); if (line == null) { - if (forever) { - if (LuceneTestCase.VERBOSE) { - System.out.println("TEST: LineFileDocs: now rewind file..."); - } - close(); - open(); - line = reader.readLine(); + // Always rewind at end: + if (LuceneTestCase.VERBOSE) { + System.out.println("TEST: LineFileDocs: now rewind file..."); } - return null; + close(); + open(null); + line = reader.readLine(); } } diff --git a/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java b/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java index 72a68709713..df4b54cda8f 100644 --- a/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java +++ b/lucene/src/test/org/apache/lucene/util/automaton/fst/TestFSTs.java @@ -944,7 +944,7 @@ public class TestFSTs extends LuceneTestCase { CodecProvider.getDefault().setDefaultFieldCodec("Standard"); } - final LineFileDocs docs = new LineFileDocs(false); + final LineFileDocs docs = new LineFileDocs(random); final int RUN_TIME_SEC = LuceneTestCase.TEST_NIGHTLY ? 100 : 1; final IndexWriterConfig conf = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxBufferedDocs(-1).setRAMBufferSizeMB(64); final File tempDir = _TestUtil.getTempDir("fstlines");