From 92b07b725e63d385a27c1f108946639ef0523943 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Mon, 20 Jun 2011 14:28:24 +0000 Subject: [PATCH] LUCENE-2793: initial commit from patch git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/LUCENE2793@1137639 13f79535-47bb-0310-9956-ffa450edef68 --- .../codecs/appending/AppendingCodec.java | 5 +- .../AppendingSegmentInfosReader.java | 5 +- .../AppendingSegmentInfosWriter.java | 5 +- .../appending/AppendingTermsDictReader.java | 5 +- .../appending/AppendingTermsIndexReader.java | 3 +- .../lucene/store/DirectIOLinuxDirectory.java | 9 ++- .../lucene/store/NRTCachingDirectory.java | 34 ++++---- .../apache/lucene/store/WindowsDirectory.java | 7 +- .../codecs/appending/TestAppendingCodec.java | 5 +- .../lucene/store/TestNRTCachingDirectory.java | 3 +- .../lucene/index/BufferedDeletesStream.java | 6 +- .../org/apache/lucene/index/CheckIndex.java | 6 +- .../lucene/index/CompoundFileReader.java | 27 ++----- .../lucene/index/CompoundFileWriter.java | 22 ++--- .../apache/lucene/index/DirectoryReader.java | 8 +- .../index/DocumentsWriterPerThread.java | 3 +- .../org/apache/lucene/index/FieldInfos.java | 4 +- .../org/apache/lucene/index/FieldsReader.java | 12 +-- .../org/apache/lucene/index/FieldsWriter.java | 6 +- .../org/apache/lucene/index/IOContext.java | 73 +++++++++++++++++ .../org/apache/lucene/index/IndexReader.java | 6 +- .../org/apache/lucene/index/IndexWriter.java | 61 ++++++++------ .../org/apache/lucene/index/MergeInfo.java | 35 ++++++++ .../org/apache/lucene/index/MergePolicy.java | 4 + .../org/apache/lucene/index/NormsWriter.java | 3 +- .../lucene/index/PerFieldCodecWrapper.java | 12 +-- .../lucene/index/SegmentCoreReaders.java | 18 ++--- .../org/apache/lucene/index/SegmentInfo.java | 4 +- .../org/apache/lucene/index/SegmentInfos.java | 13 +-- .../apache/lucene/index/SegmentMerger.java | 19 +++-- .../org/apache/lucene/index/SegmentNorms.java | 4 +- .../apache/lucene/index/SegmentReadState.java | 10 +-- .../apache/lucene/index/SegmentReader.java | 30 +++---- .../lucene/index/SegmentWriteState.java | 6 +- .../lucene/index/StoredFieldsWriter.java | 4 +- .../lucene/index/TermVectorsReader.java | 19 ++--- .../lucene/index/TermVectorsTermsWriter.java | 8 +- .../lucene/index/TermVectorsWriter.java | 8 +- .../lucene/index/codecs/BlockTermsReader.java | 5 +- .../lucene/index/codecs/BlockTermsWriter.java | 2 +- .../codecs/DefaultDocValuesProducer.java | 3 + .../codecs/DefaultSegmentInfosReader.java | 14 ++-- .../codecs/DefaultSegmentInfosWriter.java | 11 ++- .../codecs/FixedGapTermsIndexReader.java | 6 +- .../codecs/FixedGapTermsIndexWriter.java | 2 +- .../index/codecs/SegmentInfosReader.java | 3 +- .../index/codecs/SegmentInfosWriter.java | 3 +- .../codecs/VariableGapTermsIndexReader.java | 6 +- .../codecs/VariableGapTermsIndexWriter.java | 2 +- .../index/codecs/memory/MemoryCodec.java | 5 +- .../index/codecs/preflex/PreFlexCodec.java | 2 +- .../index/codecs/preflex/PreFlexFields.java | 17 ++-- .../index/codecs/preflex/TermInfosReader.java | 7 +- .../index/codecs/pulsing/PulsingCodec.java | 6 +- .../index/codecs/sep/IntStreamFactory.java | 7 +- .../codecs/sep/SepPostingsReaderImpl.java | 13 +-- .../codecs/sep/SepPostingsWriterImpl.java | 4 +- .../simpletext/SimpleTextFieldsReader.java | 2 +- .../simpletext/SimpleTextFieldsWriter.java | 2 +- .../index/codecs/standard/StandardCodec.java | 6 +- .../standard/StandardPostingsReader.java | 7 +- .../standard/StandardPostingsWriter.java | 4 +- .../org/apache/lucene/index/values/Bytes.java | 17 ++-- .../index/values/FixedDerefBytesImpl.java | 6 +- .../index/values/FixedSortedBytesImpl.java | 6 +- .../index/values/FixedStraightBytesImpl.java | 6 +- .../apache/lucene/index/values/Floats.java | 17 ++-- .../org/apache/lucene/index/values/Ints.java | 6 +- .../apache/lucene/index/values/IntsImpl.java | 9 ++- .../index/values/VarDerefBytesImpl.java | 6 +- .../index/values/VarSortedBytesImpl.java | 6 +- .../index/values/VarStraightBytesImpl.java | 6 +- .../lucene/store/BufferedIndexInput.java | 3 +- .../org/apache/lucene/store/Directory.java | 17 ++-- .../org/apache/lucene/store/FSDirectory.java | 10 +-- .../lucene/store/FileSwitchDirectory.java | 10 ++- .../apache/lucene/store/MMapDirectory.java | 4 +- .../apache/lucene/store/NIOFSDirectory.java | 10 ++- .../org/apache/lucene/store/RAMDirectory.java | 16 ++-- .../lucene/store/SimpleFSDirectory.java | 11 ++- .../org/apache/lucene/util/BitVector.java | 11 ++- .../mockintblock/MockFixedIntBlockCodec.java | 14 ++-- .../MockVariableIntBlockCodec.java | 13 +-- .../codecs/mockrandom/MockRandomCodec.java | 19 ++--- .../index/codecs/mocksep/MockSepCodec.java | 6 +- .../codecs/mocksep/MockSingleIntFactory.java | 5 +- .../mocksep/MockSingleIntIndexInput.java | 5 +- .../mocksep/MockSingleIntIndexOutput.java | 4 +- .../codecs/preflexrw/PreFlexFieldsWriter.java | 4 +- .../codecs/preflexrw/PreFlexRWCodec.java | 2 +- .../codecs/preflexrw/TermInfosWriter.java | 4 +- .../lucene/store/MockDirectoryWrapper.java | 22 +++-- .../apache/lucene/util/LuceneTestCase.java | 3 +- .../apache/lucene/index/TestAddIndexes.java | 6 +- .../index/TestBackwardsCompatibility.java | 3 +- .../org/apache/lucene/index/TestCodecs.java | 7 +- .../apache/lucene/index/TestCompoundFile.java | 80 ++++++++++--------- .../test/org/apache/lucene/index/TestDoc.java | 11 ++- .../apache/lucene/index/TestDocTermOrds.java | 6 +- .../lucene/index/TestDocumentWriter.java | 9 ++- .../apache/lucene/index/TestFieldInfos.java | 3 +- .../apache/lucene/index/TestFieldsReader.java | 8 +- .../lucene/index/TestIndexFileDeleter.java | 6 +- .../apache/lucene/index/TestIndexInput.java | 4 +- .../index/TestIndexReaderOnDiskFull.java | 2 +- .../apache/lucene/index/TestIndexWriter.java | 2 +- .../lucene/index/TestIndexWriterDelete.java | 2 +- .../index/TestIndexWriterExceptions.java | 14 ++-- .../index/TestIndexWriterOnDiskFull.java | 2 +- .../lucene/index/TestIndexWriterReader.java | 4 +- .../lucene/index/TestLazyProxSkipping.java | 4 +- .../lucene/index/TestMultiLevelSkipList.java | 4 +- .../apache/lucene/index/TestMultiReader.java | 4 +- .../lucene/index/TestSegmentMerger.java | 10 +-- .../lucene/index/TestSegmentReader.java | 5 +- .../lucene/index/TestSegmentTermDocs.java | 6 +- .../index/TestSnapshotDeletionPolicy.java | 2 +- .../lucene/index/TestTermVectorsReader.java | 15 ++-- .../lucene/index/TestTermVectorsWriter.java | 2 +- .../codecs/intblock/TestIntBlockCodec.java | 5 +- .../apache/lucene/search/TestBoolean2.java | 5 +- .../lucene/store/TestBufferedIndexInput.java | 20 ++--- .../apache/lucene/store/TestCopyBytes.java | 9 ++- .../apache/lucene/store/TestDirectory.java | 13 +-- .../apache/lucene/store/TestMultiMMap.java | 13 +-- .../apache/lucene/store/TestRAMDirectory.java | 13 +-- .../org/apache/lucene/util/TestBitVector.java | 15 ++-- .../org/apache/lucene/util/fst/TestFSTs.java | 11 +-- .../lucene/util/packed/TestPackedInts.java | 13 +-- .../apache/solr/core/RefCntRamDirectory.java | 3 +- 130 files changed, 746 insertions(+), 524 deletions(-) create mode 100644 lucene/src/java/org/apache/lucene/index/IOContext.java create mode 100644 lucene/src/java/org/apache/lucene/index/MergeInfo.java diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java index 77b2eaf38eb..2d1617bea4c 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingCodec.java @@ -94,11 +94,12 @@ public class AppendingCodec extends Codec { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); TermsIndexReaderBase indexReader; boolean success = false; try { + //nocommit we should pass the IOContext down to the TermIndexReader indexReader = new AppendingTermsIndexReader(state.dir, state.fieldInfos, state.segmentInfo.name, @@ -116,7 +117,7 @@ public class AppendingCodec extends Codec { FieldsProducer ret = new AppendingTermsDictReader(indexReader, state.dir, state.fieldInfos, state.segmentInfo.name, docsReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java index bd4b26c5c9a..65217972232 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosReader.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.appending; import java.io.IOException; import org.apache.lucene.index.CorruptIndexException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.DefaultSegmentInfosReader; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -33,9 +34,9 @@ public class AppendingSegmentInfosReader extends DefaultSegmentInfosReader { } @Override - public IndexInput openInput(Directory dir, String segmentsFileName) + public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context) throws IOException { - return dir.openInput(segmentsFileName); + return dir.openInput(segmentsFileName, context); } } diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java index 45d53e01955..0a6ee864cbe 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingSegmentInfosWriter.java @@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs.appending; import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -26,9 +27,9 @@ import org.apache.lucene.store.IndexOutput; public class AppendingSegmentInfosWriter extends DefaultSegmentInfosWriter { @Override - protected IndexOutput createOutput(Directory dir, String segmentsFileName) + protected IndexOutput createOutput(Directory dir, String segmentsFileName, IOContext context) throws IOException { - return dir.createOutput(segmentsFileName); + return dir.createOutput(segmentsFileName, context); } @Override diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java index b12c4f8ad08..9c10ddaf91e 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsDictReader.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.appending; import java.io.IOException; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.PostingsReaderBase; import org.apache.lucene.index.codecs.BlockTermsReader; import org.apache.lucene.index.codecs.BlockTermsWriter; @@ -32,9 +33,9 @@ public class AppendingTermsDictReader extends BlockTermsReader { public AppendingTermsDictReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, - PostingsReaderBase postingsReader, int readBufferSize, + PostingsReaderBase postingsReader, IOContext context, int termsCacheSize, int codecId) throws IOException { - super(indexReader, dir, fieldInfos, segment, postingsReader, readBufferSize, + super(indexReader, dir, fieldInfos, segment, postingsReader, context, termsCacheSize, codecId); } diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java index 0a449707e11..20ecbe7f7bf 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/index/codecs/appending/AppendingTermsIndexReader.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Comparator; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.FixedGapTermsIndexReader; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -32,7 +33,7 @@ public class AppendingTermsIndexReader extends FixedGapTermsIndexReader { public AppendingTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId) throws IOException { - super(dir, fieldInfos, segment, indexDivisor, termComp, codecId); + super(dir, fieldInfos, segment, indexDivisor, termComp, codecId, IOContext.DEFAULT); } @Override diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java b/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java index 93ace13ecbb..9f3bc1d10a3 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java @@ -25,6 +25,7 @@ import java.io.FileOutputStream; import java.nio.ByteBuffer; import java.nio.channels.FileChannel; +import org.apache.lucene.index.IOContext; import org.apache.lucene.store.Directory; // javadoc import org.apache.lucene.store.NativeFSLockFactory; // javadoc @@ -69,15 +70,17 @@ public class DirectIOLinuxDirectory extends FSDirectory { } @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? bufferSize : forcedBufferSize); + //nocommit - use buffer based on IOContext + return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexInput.BUFFER_SIZE : forcedBufferSize); } @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name,IOContext context) throws IOException { ensureOpen(); ensureCanWrite(name); + //nocommit - use buffer based on IOContext return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexOutput.BUFFER_SIZE : forcedBufferSize); } diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java b/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java index 6cd8ac02cba..89de60561d7 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/store/NRTCachingDirectory.java @@ -24,8 +24,10 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import org.apache.lucene.index.ConcurrentMergeScheduler; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexWriter; // javadocs +import org.apache.lucene.index.MergeInfo; import org.apache.lucene.index.MergePolicy; import org.apache.lucene.index.MergeScheduler; import org.apache.lucene.store.RAMDirectory; // javadocs @@ -193,17 +195,17 @@ public class NRTCachingDirectory extends Directory { } @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.createOutput name=" + name); } - if (doCacheWrite(name)) { + if (doCacheWrite(name, context)) { if (VERBOSE) { System.out.println(" to cache"); } - return cache.createOutput(name); + return cache.createOutput(name, context); } else { - return delegate.createOutput(name); + return delegate.createOutput(name, context); } } @@ -219,7 +221,7 @@ public class NRTCachingDirectory extends Directory { } @Override - public synchronized IndexInput openInput(String name) throws IOException { + public synchronized IndexInput openInput(String name, IOContext context) throws IOException { if (VERBOSE) { System.out.println("nrtdir.openInput name=" + name); } @@ -227,18 +229,9 @@ public class NRTCachingDirectory extends Directory { if (VERBOSE) { System.out.println(" from cache"); } - return cache.openInput(name); + return cache.openInput(name, context); } else { - return delegate.openInput(name); - } - } - - @Override - public synchronized IndexInput openInput(String name, int bufferSize) throws IOException { - if (cache.fileExists(name)) { - return cache.openInput(name, bufferSize); - } else { - return delegate.openInput(name, bufferSize); + return delegate.openInput(name, context); } } @@ -271,18 +264,19 @@ public class NRTCachingDirectory extends Directory { /** Subclass can override this to customize logic; return * true if this file should be written to the RAMDirectory. */ - protected boolean doCacheWrite(String name) { - final MergePolicy.OneMerge merge = merges.get(Thread.currentThread()); + protected boolean doCacheWrite(String name, IOContext context) { + final MergeInfo merge = context.mergeInfo; //System.out.println(Thread.currentThread().getName() + ": CACHE check merge=" + merge + " size=" + (merge==null ? 0 : merge.estimatedMergeBytes)); return !name.equals(IndexFileNames.SEGMENTS_GEN) && (merge == null || merge.estimatedMergeBytes <= maxMergeSizeBytes) && cache.sizeInBytes() <= maxCachedBytes; } private void unCache(String fileName) throws IOException { final IndexOutput out; + IOContext context = IOContext.DEFAULT; synchronized(this) { if (!delegate.fileExists(fileName)) { assert cache.fileExists(fileName); - out = delegate.createOutput(fileName); + out = delegate.createOutput(fileName, context); } else { out = null; } @@ -291,7 +285,7 @@ public class NRTCachingDirectory extends Directory { if (out != null) { IndexInput in = null; try { - in = cache.openInput(fileName); + in = cache.openInput(fileName, context); in.copyBytes(out, in.length()); } finally { IOUtils.closeSafely(false, in, out); diff --git a/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java b/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java index a4c6301ee34..1f34755422e 100644 --- a/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java +++ b/lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java @@ -19,6 +19,8 @@ package org.apache.lucene.store; import java.io.File; import java.io.IOException; + +import org.apache.lucene.index.IOContext; import org.apache.lucene.store.Directory; // javadoc import org.apache.lucene.store.NativeFSLockFactory; // javadoc @@ -67,9 +69,10 @@ public class WindowsDirectory extends FSDirectory { } @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new WindowsIndexInput(new File(getDirectory(), name), Math.max(bufferSize, DEFAULT_BUFFERSIZE)); + //nocommit - use buffer based on IOContext + return new WindowsIndexInput(new File(getDirectory(), name), DEFAULT_BUFFERSIZE); } protected static class WindowsIndexInput extends BufferedIndexInput { diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java index 125cc1847a9..d127685089b 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java @@ -27,6 +27,7 @@ import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field.TermVector; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.Fields; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -124,8 +125,8 @@ public class TestAppendingCodec extends LuceneTestCase { } @Override - public IndexOutput createOutput(String name) throws IOException { - return new AppendingIndexOutputWrapper(super.createOutput(name)); + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return new AppendingIndexOutputWrapper(super.createOutput(name, context)); } } diff --git a/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java b/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java index 9001d13a4de..364d327e03c 100644 --- a/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java +++ b/lucene/contrib/misc/src/test/org/apache/lucene/store/TestNRTCachingDirectory.java @@ -24,6 +24,7 @@ import java.util.List; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -114,7 +115,7 @@ public class TestNRTCachingDirectory extends LuceneTestCase { public void testDeleteFile() throws Exception { Directory dir = new NRTCachingDirectory(newDirectory(), 2.0, 25.0); - dir.createOutput("foo.txt").close(); + dir.createOutput("foo.txt", IOContext.DEFAULT).close(); dir.deleteFile("foo.txt"); assertEquals(0, dir.listAll().length); dir.close(); diff --git a/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java b/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java index 6a9303c3c80..800fad98321 100644 --- a/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java +++ b/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java @@ -224,7 +224,8 @@ class BufferedDeletesStream { // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); - final SegmentReader reader = readerPool.get(info, false); + //nocommit is IOContext.DEFAULT the right thing to do here? + final SegmentReader reader = readerPool.get(info, false, IOContext.DEFAULT); int delCount = 0; final boolean segAllDeletes; try { @@ -273,7 +274,8 @@ class BufferedDeletesStream { if (coalescedDeletes != null) { // Lock order: IW -> BD -> RP assert readerPool.infoIsLive(info); - SegmentReader reader = readerPool.get(info, false); + //nocommit is IOContext.DEFAULT the right thing to do here? + SegmentReader reader = readerPool.get(info, false, IOContext.DEFAULT); int delCount = 0; final boolean segAllDeletes; try { diff --git a/lucene/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/src/java/org/apache/lucene/index/CheckIndex.java index 6b075d8b73d..f037ce730f1 100644 --- a/lucene/src/java/org/apache/lucene/index/CheckIndex.java +++ b/lucene/src/java/org/apache/lucene/index/CheckIndex.java @@ -25,6 +25,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.document.AbstractField; // for javadocs import org.apache.lucene.document.Document; +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.PerDocValues; @@ -342,7 +344,7 @@ public class CheckIndex { final String segmentsFileName = sis.getCurrentSegmentFileName(); IndexInput input = null; try { - input = dir.openInput(segmentsFileName); + input = dir.openInput(segmentsFileName, IOContext.DEFAULT); } catch (Throwable t) { msg("ERROR: could not open segments file in directory"); if (infoStream != null) @@ -479,7 +481,7 @@ public class CheckIndex { } if (infoStream != null) infoStream.print(" test: open reader........."); - reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.DEFAULT); segInfoStat.openReaderPassed = true; diff --git a/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java b/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java index e38fc7bafa7..1af43b19887 100644 --- a/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java +++ b/lucene/src/java/org/apache/lucene/index/CompoundFileReader.java @@ -36,8 +36,6 @@ import java.io.IOException; */ public class CompoundFileReader extends Directory { - private int readBufferSize; - private static final class FileEntry { long offset; long length; @@ -49,21 +47,16 @@ public class CompoundFileReader extends Directory { private IndexInput stream; private HashMap entries = new HashMap(); - - public CompoundFileReader(Directory dir, String name) throws IOException { - this(dir, name, BufferedIndexInput.BUFFER_SIZE); - } - - public CompoundFileReader(Directory dir, String name, int readBufferSize) throws IOException { + + public CompoundFileReader(Directory dir, String name, IOContext context) throws IOException { assert !(dir instanceof CompoundFileReader) : "compound file inside of compound file: " + name; directory = dir; fileName = name; - this.readBufferSize = readBufferSize; boolean success = false; try { - stream = dir.openInput(name, readBufferSize); + stream = dir.openInput(name, context); // read the first VInt. If it is negative, it's the version number // otherwise it's the count (pre-3.1 indexes) @@ -141,13 +134,7 @@ public class CompoundFileReader extends Directory { } @Override - public synchronized IndexInput openInput(String id) throws IOException { - // Default to readBufferSize passed in when we were opened - return openInput(id, readBufferSize); - } - - @Override - public synchronized IndexInput openInput(String id, int readBufferSize) throws IOException { + public synchronized IndexInput openInput(String id, IOContext context) throws IOException { if (stream == null) throw new IOException("Stream closed"); @@ -155,8 +142,8 @@ public class CompoundFileReader extends Directory { final FileEntry entry = entries.get(id); if (entry == null) throw new IOException("No sub-file with id " + id + " found (files: " + entries.keySet() + ")"); - - return new CSIndexInput(stream, entry.offset, entry.length, readBufferSize); + // nocommit set read buffer size based on IOContext + return new CSIndexInput(stream, entry.offset, entry.length, BufferedIndexInput.BUFFER_SIZE); } /** Returns an array of strings, one for each file in the directory. */ @@ -209,7 +196,7 @@ public class CompoundFileReader extends Directory { /** Not implemented * @throws UnsupportedOperationException */ @Override - public IndexOutput createOutput(String name) { + public IndexOutput createOutput(String name, IOContext context) { throw new UnsupportedOperationException(); } diff --git a/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java b/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java index e47fed7b717..451bb4a1629 100644 --- a/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java +++ b/lucene/src/java/org/apache/lucene/index/CompoundFileWriter.java @@ -76,22 +76,23 @@ public final class CompoundFileWriter { // switch to a new format! static final int FORMAT_CURRENT = FORMAT_NO_SEGMENT_PREFIX; - private Directory directory; - private String fileName; - private HashSet ids; - private LinkedList entries; + private final Directory directory; + private final String fileName; + private final IOContext context; + private final HashSet ids; + private final LinkedList entries; private boolean merged = false; - private MergeState.CheckAbort checkAbort; + private final MergeState.CheckAbort checkAbort; /** Create the compound stream in the specified file. The file name is the * entire name (no extensions are added). * @throws NullPointerException if dir or name is null */ - public CompoundFileWriter(Directory dir, String name) { - this(dir, name, null); + public CompoundFileWriter(Directory dir, String name, IOContext context) { + this(dir, name, context, null); } - CompoundFileWriter(Directory dir, String name, MergeState.CheckAbort checkAbort) { + CompoundFileWriter(Directory dir, String name, IOContext context, MergeState.CheckAbort checkAbort) { if (dir == null) throw new NullPointerException("directory cannot be null"); if (name == null) @@ -101,6 +102,7 @@ public final class CompoundFileWriter { fileName = name; ids = new HashSet(); entries = new LinkedList(); + this.context = context; } /** Returns the directory of the compound file. */ @@ -164,7 +166,7 @@ public final class CompoundFileWriter { merged = true; // open the compound stream - IndexOutput os = directory.createOutput(fileName); + IndexOutput os = directory.createOutput(fileName, context); IOException priorException = null; try { // Write the Version info - must be a VInt because CFR reads a VInt @@ -228,7 +230,7 @@ public final class CompoundFileWriter { * output stream. */ private void copyFile(FileEntry source, IndexOutput os) throws IOException { - IndexInput is = source.dir.openInput(source.file); + IndexInput is = source.dir.openInput(source.file, context); try { long startPtr = os.getFilePointer(); long length = is.length(); diff --git a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java index 2b05adf7030..7b3132d24ce 100644 --- a/lucene/src/java/org/apache/lucene/index/DirectoryReader.java +++ b/lucene/src/java/org/apache/lucene/index/DirectoryReader.java @@ -34,6 +34,7 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.store.Directory; import org.apache.lucene.store.Lock; import org.apache.lucene.store.LockObtainFailedException; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.PerDocValues; import org.apache.lucene.util.Bits; @@ -121,7 +122,7 @@ class DirectoryReader extends IndexReader implements Cloneable { for (int i = sis.size()-1; i >= 0; i--) { boolean success = false; try { - readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor); + readers[i] = SegmentReader.get(readOnly, sis.info(i), termInfosIndexDivisor, IOContext.READ); readers[i].readerFinishedListeners = readerFinishedListeners; success = true; } finally { @@ -170,7 +171,8 @@ class DirectoryReader extends IndexReader implements Cloneable { try { final SegmentInfo info = infos.info(i); assert info.dir == dir; - final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor); + final SegmentReader reader = writer.readerPool.getReadOnlyClone(info, true, termInfosIndexDivisor, + IOContext.READ); if (reader.numDocs() > 0 || writer.getKeepFullyDeletedSegments()) { reader.readerFinishedListeners = readerFinishedListeners; readers.add(reader); @@ -254,7 +256,7 @@ class DirectoryReader extends IndexReader implements Cloneable { assert !doClone; // this is a new reader; in case we hit an exception we can close it safely - newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor); + newReader = SegmentReader.get(readOnly, infos.info(i), termInfosIndexDivisor, IOContext.READ); newReader.readerFinishedListeners = readerFinishedListeners; } else { newReader = newReaders[i].reopenSegment(infos.info(i), doClone, readOnly); diff --git a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java index baf5d3bf43b..2a12b2f812f 100644 --- a/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java +++ b/lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java @@ -28,6 +28,7 @@ import java.util.concurrent.atomic.AtomicLong; import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DocumentsWriterDeleteQueue.DeleteSlice; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.search.SimilarityProvider; import org.apache.lucene.store.Directory; import org.apache.lucene.util.BitVector; @@ -428,7 +429,7 @@ public class DocumentsWriterPerThread { assert deleteSlice == null : "all deletes must be applied in prepareFlush"; flushState = new SegmentWriteState(infoStream, directory, segment, fieldInfos, numDocsInRAM, writer.getConfig().getTermIndexInterval(), - fieldInfos.buildSegmentCodecs(true), pendingDeletes); + fieldInfos.buildSegmentCodecs(true), pendingDeletes, new IOContext(Context.FLUSH)); final double startMBUsed = parent.flushControl.netBytes() / 1024. / 1024.; // Apply delete-by-docID now (delete-byDocID only // happens when an exception is hit processing that diff --git a/lucene/src/java/org/apache/lucene/index/FieldInfos.java b/lucene/src/java/org/apache/lucene/index/FieldInfos.java index 594301326f4..b98af343468 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldInfos.java +++ b/lucene/src/java/org/apache/lucene/index/FieldInfos.java @@ -271,7 +271,7 @@ public final class FieldInfos implements Iterable { */ public FieldInfos(Directory d, String name) throws IOException { this((FieldNumberBiMap)null, null); // use null here to make this FIs Read-Only - final IndexInput input = d.openInput(name); + final IndexInput input = d.openInput(name, IOContext.READONCE); try { read(input, name); } finally { @@ -564,7 +564,7 @@ public final class FieldInfos implements Iterable { } public void write(Directory d, String name) throws IOException { - IndexOutput output = d.createOutput(name); + IndexOutput output = d.createOutput(name, IOContext.READONCE); try { write(output); } finally { diff --git a/lucene/src/java/org/apache/lucene/index/FieldsReader.java b/lucene/src/java/org/apache/lucene/index/FieldsReader.java index e135d6d2870..01348af0c87 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/FieldsReader.java @@ -25,6 +25,8 @@ import org.apache.lucene.document.FieldSelector; import org.apache.lucene.document.FieldSelectorResult; import org.apache.lucene.document.Fieldable; import org.apache.lucene.document.NumericField; +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.AlreadyClosedException; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; @@ -81,7 +83,7 @@ public final class FieldsReader implements Cloneable { /** Verifies that the code version which wrote the segment is supported. */ public static void checkCodeVersion(Directory dir, String segment) throws IOException { final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); - IndexInput idxStream = dir.openInput(indexStreamFN, 1024); + IndexInput idxStream = dir.openInput(indexStreamFN, IOContext.DEFAULT); try { int format = idxStream.readInt(); @@ -110,18 +112,18 @@ public final class FieldsReader implements Cloneable { } public FieldsReader(Directory d, String segment, FieldInfos fn) throws IOException { - this(d, segment, fn, BufferedIndexInput.BUFFER_SIZE, -1, 0); + this(d, segment, fn, IOContext.DEFAULT, -1, 0); } - public FieldsReader(Directory d, String segment, FieldInfos fn, int readBufferSize, int docStoreOffset, int size) throws IOException { + public FieldsReader(Directory d, String segment, FieldInfos fn, IOContext context, int docStoreOffset, int size) throws IOException { boolean success = false; isOriginal = true; try { fieldInfos = fn; - cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), readBufferSize); + cloneableFieldsStream = d.openInput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); final String indexStreamFN = IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION); - cloneableIndexStream = d.openInput(indexStreamFN, readBufferSize); + cloneableIndexStream = d.openInput(indexStreamFN, context); format = cloneableIndexStream.readInt(); diff --git a/lucene/src/java/org/apache/lucene/index/FieldsWriter.java b/lucene/src/java/org/apache/lucene/index/FieldsWriter.java index 5542acff892..649aa54571b 100644 --- a/lucene/src/java/org/apache/lucene/index/FieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/FieldsWriter.java @@ -65,14 +65,14 @@ final class FieldsWriter { private IndexOutput fieldsStream; private IndexOutput indexStream; - FieldsWriter(Directory directory, String segment) throws IOException { + FieldsWriter(Directory directory, String segment, IOContext context) throws IOException { this.directory = directory; this.segment = segment; boolean success = false; try { - fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION)); - indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION)); + fieldsStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_EXTENSION), context); + indexStream = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.FIELDS_INDEX_EXTENSION), context); fieldsStream.writeInt(FORMAT_CURRENT); indexStream.writeInt(FORMAT_CURRENT); diff --git a/lucene/src/java/org/apache/lucene/index/IOContext.java b/lucene/src/java/org/apache/lucene/index/IOContext.java new file mode 100644 index 00000000000..5e5127d8517 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/IOContext.java @@ -0,0 +1,73 @@ +package org.apache.lucene.index; + +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * +* Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * IOContext holds additional details on the merge/search context. A IOContext + * object can never be initialized as null as passed as a parameter to either + * {@link #org.apache.lucene.store.Directory.openInput()} or + * {@link #org.apache.lucene.store.Directory.createInput()} + */ +public class IOContext { + + /** + * Context is a enumerator which specifies the context in which the Directory is being used for. + */ + public enum Context {MERGE,READ,FLUSH,DEFAULT}; + + /** + * An object of a enumerator Context type + */ + public final Context context; + + public final MergeInfo mergeInfo; + + public final boolean readOnce; + + public static final IOContext DEFAULT = new IOContext(Context.DEFAULT); + + public static final IOContext READONCE = new IOContext(true); + + public static final IOContext READ = new IOContext(false); + + public IOContext () { + this(false); + } + + public IOContext(Context context) { + this(context, null); + } + + private IOContext(boolean readOnce) { + this.context = Context.READ; + this.mergeInfo = null; + this.readOnce = readOnce; + } + + public IOContext (MergeInfo mergeInfo) { + this(Context.MERGE, mergeInfo); + } + + private IOContext (Context context, MergeInfo mergeInfo ) { + assert context != Context.MERGE || mergeInfo != null; + this.context = context; + this.readOnce = false; + this.mergeInfo = mergeInfo; + } + +} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/IndexReader.java b/lucene/src/java/org/apache/lucene/index/IndexReader.java index 9d229dfcf0e..eacfd083112 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/IndexReader.java @@ -21,6 +21,7 @@ import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.search.FieldCache; // javadocs import org.apache.lucene.search.Similarity; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.codecs.Codec; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.PerDocValues; @@ -1433,13 +1434,14 @@ public abstract class IndexReader implements Cloneable,Closeable { Directory dir = null; CompoundFileReader cfr = null; + IOContext context = IOContext.READ; try { File file = new File(filename); String dirname = file.getAbsoluteFile().getParent(); filename = file.getName(); dir = FSDirectory.open(new File(dirname)); - cfr = new CompoundFileReader(dir, filename); + cfr = new CompoundFileReader(dir, filename, context); String [] files = cfr.listAll(); ArrayUtil.mergeSort(files); // sort the array of filename so that the output is more readable @@ -1449,7 +1451,7 @@ public abstract class IndexReader implements Cloneable,Closeable { if (extract) { System.out.println("extract " + files[i] + " with " + len + " bytes to local directory..."); - IndexInput ii = cfr.openInput(files[i]); + IndexInput ii = cfr.openInput(files[i], context); FileOutputStream f = new FileOutputStream(files[i]); diff --git a/lucene/src/java/org/apache/lucene/index/IndexWriter.java b/lucene/src/java/org/apache/lucene/index/IndexWriter.java index 8817c7601bd..aca12e5ac40 100644 --- a/lucene/src/java/org/apache/lucene/index/IndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/IndexWriter.java @@ -39,7 +39,9 @@ import org.apache.lucene.analysis.Analyzer; import org.apache.lucene.document.Document; import org.apache.lucene.index.DocumentsWriterPerThread.FlushedSegment; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexWriterConfig.OpenMode; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.index.PayloadProcessorProvider.DirPayloadProcessor; import org.apache.lucene.index.SegmentCodecs.SegmentCodecsBuilder; import org.apache.lucene.index.codecs.CodecProvider; @@ -592,8 +594,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * enrolled in the pool, so you should simply close() * it when you're done (ie, do not call release()). */ - public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor) throws IOException { - SegmentReader sr = get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, termInfosIndexDivisor); + public synchronized SegmentReader getReadOnlyClone(SegmentInfo info, boolean doOpenStores, int termInfosIndexDivisor, IOContext context) throws IOException { + SegmentReader sr = get(info, doOpenStores, context, termInfosIndexDivisor); try { return (SegmentReader) sr.clone(true); } finally { @@ -609,8 +611,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @param doOpenStores * @throws IOException */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores) throws IOException { - return get(info, doOpenStores, BufferedIndexInput.BUFFER_SIZE, config.getReaderTermsIndexDivisor()); + public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, IOContext context) throws IOException { + return get(info, doOpenStores, context, config.getReaderTermsIndexDivisor()); } /** @@ -624,18 +626,20 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { * @param termsIndexDivisor * @throws IOException */ - public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, int readBufferSize, int termsIndexDivisor) throws IOException { + public synchronized SegmentReader get(SegmentInfo info, boolean doOpenStores, IOContext context, int termsIndexDivisor) throws IOException { - if (poolReaders) { - readBufferSize = BufferedIndexInput.BUFFER_SIZE; - } + // if (poolReaders) { + // readBufferSize = BufferedIndexInput.BUFFER_SIZE; + // } + + // nocommit context should be part of the key used to cache that reader in the pool. SegmentReader sr = readerMap.get(info); if (sr == null) { // TODO: we may want to avoid doing this while // synchronized // Returns a ref, which we xfer to readerMap: - sr = SegmentReader.get(false, info.dir, info, readBufferSize, doOpenStores, termsIndexDivisor); + sr = SegmentReader.get(false, info.dir, info, doOpenStores, termsIndexDivisor, context); sr.readerFinishedListeners = readerFinishedListeners; if (info.dir == directory) { @@ -2183,6 +2187,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { SegmentInfo newSegment = flushedSegment.segmentInfo; setDiagnostics(newSegment, "flush"); + + IOContext context = new IOContext(Context.FLUSH); boolean success = false; try { @@ -2190,7 +2196,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { String compoundFileName = IndexFileNames.segmentFileName(newSegment.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION); message("creating compound file " + compoundFileName); // Now build compound file - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName); + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, compoundFileName, context); for(String fileName : newSegment.files()) { cfsWriter.addFile(fileName); } @@ -2222,7 +2228,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // shortly-to-be-opened SegmentReader and let it // carry the changes; there's no reason to use // filesystem as intermediary here. - flushedSegment.deletedDocuments.write(directory, delFileName); + flushedSegment.deletedDocuments.write(directory, delFileName, context); success2 = true; } finally { if (!success2) { @@ -2391,11 +2397,13 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // optimize case only for segments that don't share doc stores && versionComparator.compare(info.getVersion(), "3.1") >= 0; } - + + IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(true), true, false)); + if (createCFS) { - copySegmentIntoCFS(info, newSegName); + copySegmentIntoCFS(info, newSegName, context); } else { - copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied); + copySegmentAsIs(info, newSegName, dsNames, dsFilesCopied, context); } infos.add(info); @@ -2454,7 +2462,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { for (IndexReader reader : readers) // add new indexes merger.add(reader); - + // nocommit - we should pass a MergeInfo here into merge to create corresponding IOContext instances? int docCount = merger.merge(); // merge 'em final FieldInfos fieldInfos = merger.fieldInfos(); @@ -2475,7 +2483,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Now create the compound file if needed if (useCompoundFile) { - merger.createCompoundFile(mergedName + ".cfs", info); + IOContext context = new IOContext(new MergeInfo(info.docCount, info.sizeInBytes(true), true, false)); + merger.createCompoundFile(mergedName + ".cfs", info, context); // delete new non cfs files directly: they were never // registered with IFD @@ -2499,10 +2508,10 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } /** Copies the segment into the IndexWriter's directory, as a compound segment. */ - private void copySegmentIntoCFS(SegmentInfo info, String segName) throws IOException { + private void copySegmentIntoCFS(SegmentInfo info, String segName, IOContext context) throws IOException { String segFileName = IndexFileNames.segmentFileName(segName, "", IndexFileNames.COMPOUND_FILE_EXTENSION); Collection files = info.files(); - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segFileName); + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, segFileName, context); for (String file : files) { String newFileName = segName + IndexFileNames.stripSegmentName(file); if (!IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION) @@ -2510,7 +2519,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { cfsWriter.addFile(file, info.dir); } else { assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists"; - info.dir.copy(directory, file, newFileName); + info.dir.copy(directory, file, newFileName, context); } } @@ -2524,7 +2533,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { /** Copies the segment files as-is into the IndexWriter's directory. */ private void copySegmentAsIs(SegmentInfo info, String segName, - Map dsNames, Set dsFilesCopied) + Map dsNames, Set dsFilesCopied, IOContext context) throws IOException { // Determine if the doc store of this segment needs to be copied. It's // only relevant for segments that share doc store with others, @@ -2560,7 +2569,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { } assert !directory.fileExists(newFileName): "file \"" + newFileName + "\" already exists"; - info.dir.copy(directory, file, newFileName); + info.dir.copy(directory, file, newFileName, context); } info.setDocStore(info.getDocStoreOffset(), newDsName, info.getDocStoreIsCompoundFile()); @@ -3416,6 +3425,8 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { int mergedDocCount = 0; List sourceSegments = merge.segments; + + IOContext context = new IOContext(merge.getMergeInfo()); SegmentMerger merger = new SegmentMerger(directory, config.getTermIndexInterval(), mergedName, merge, payloadProcessorProvider, merge.info.getFieldInfos()); @@ -3439,7 +3450,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // Hold onto the "live" reader; we will use this to // commit merged deletes final SegmentReader reader = readerPool.get(info, true, - MERGE_READ_BUFFER_SIZE, + context, -config.getReaderTermsIndexDivisor()); merge.readers.add(reader); @@ -3493,7 +3504,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { if (infoStream != null) { message("create compound file " + compoundFileName); } - merger.createCompoundFile(compoundFileName, merge.info); + merger.createCompoundFile(compoundFileName, merge.info, new IOContext(merge.getMergeInfo())); success = true; } catch (IOException ioe) { synchronized(this) { @@ -3563,7 +3574,9 @@ public class IndexWriter implements Closeable, TwoPhaseCommit { // keep deletes (it's costly to open entire reader // when we just need deletes) - final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, BufferedIndexInput.BUFFER_SIZE, termsIndexDivisor); + // nocommit should we use another flag "isMergedSegment" or a "READ" context here? + + final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, context, termsIndexDivisor); try { if (poolReaders && mergedSegmentWarmer != null) { mergedSegmentWarmer.warm(mergedReader); diff --git a/lucene/src/java/org/apache/lucene/index/MergeInfo.java b/lucene/src/java/org/apache/lucene/index/MergeInfo.java new file mode 100644 index 00000000000..208eeb41f94 --- /dev/null +++ b/lucene/src/java/org/apache/lucene/index/MergeInfo.java @@ -0,0 +1,35 @@ +package org.apache.lucene.index; +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +//nocommit javadoc +public class MergeInfo { + + public int totalDocCount; + + public long estimatedMergeBytes; // used by IndexWriter + + boolean isExternal; // used by IndexWriter + + boolean optimize; // used by IndexWriter + + public MergeInfo(int totalDocCount, long estimatedMergeBytes, boolean isExternal, boolean optimize) { + this.totalDocCount = totalDocCount; + this.estimatedMergeBytes = estimatedMergeBytes; + this.isExternal = isExternal; + this.optimize = optimize; + } +} \ No newline at end of file diff --git a/lucene/src/java/org/apache/lucene/index/MergePolicy.java b/lucene/src/java/org/apache/lucene/index/MergePolicy.java index 093ac8c9fb7..a6c5bf83128 100644 --- a/lucene/src/java/org/apache/lucene/index/MergePolicy.java +++ b/lucene/src/java/org/apache/lucene/index/MergePolicy.java @@ -189,6 +189,10 @@ public abstract class MergePolicy implements java.io.Closeable { } return total; } + + public MergeInfo getMergeInfo() { + return new MergeInfo(totalDocCount, estimatedMergeBytes, isExternal, optimize); + } } /** diff --git a/lucene/src/java/org/apache/lucene/index/NormsWriter.java b/lucene/src/java/org/apache/lucene/index/NormsWriter.java index 91c7eed44cf..41ed5c75afa 100644 --- a/lucene/src/java/org/apache/lucene/index/NormsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/NormsWriter.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Collection; import java.util.Map; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.IOUtils; @@ -49,7 +50,7 @@ final class NormsWriter extends InvertedDocEndConsumer { } final String normsFileName = IndexFileNames.segmentFileName(state.segmentName, "", IndexFileNames.NORMS_EXTENSION); - IndexOutput normsOut = state.directory.createOutput(normsFileName); + IndexOutput normsOut = state.directory.createOutput(normsFileName, state.context); boolean success = false; try { normsOut.writeBytes(SegmentNorms.NORMS_HEADER, 0, SegmentNorms.NORMS_HEADER.length); diff --git a/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java b/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java index 0a9a18718b1..6e52d4f90da 100644 --- a/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java +++ b/lucene/src/java/org/apache/lucene/index/PerFieldCodecWrapper.java @@ -99,7 +99,7 @@ final class PerFieldCodecWrapper extends Codec { private final Map codecs = new HashMap(); public FieldsReader(Directory dir, FieldInfos fieldInfos, SegmentInfo si, - int readBufferSize, int indexDivisor) throws IOException { + IOContext context, int indexDivisor) throws IOException { final Map producers = new HashMap(); boolean success = false; @@ -111,7 +111,7 @@ final class PerFieldCodecWrapper extends Codec { Codec codec = segmentCodecs.codecs[fi.getCodecId()]; if (!producers.containsKey(codec)) { producers.put(codec, codec.fieldsProducer(new SegmentReadState(dir, - si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId()))); + si, fieldInfos, context, indexDivisor, fi.getCodecId()))); } codecs.put(fi.name, producers.get(codec)); } @@ -187,7 +187,7 @@ final class PerFieldCodecWrapper extends Codec { public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { return new FieldsReader(state.dir, state.fieldInfos, state.segmentInfo, - state.readBufferSize, state.termsIndexDivisor); + state.context, state.termsIndexDivisor); } @Override @@ -212,14 +212,14 @@ final class PerFieldCodecWrapper extends Codec { @Override public PerDocValues docsProducer(SegmentReadState state) throws IOException { return new PerDocProducers(state.dir, state.fieldInfos, state.segmentInfo, - state.readBufferSize, state.termsIndexDivisor); + state.context, state.termsIndexDivisor); } private final class PerDocProducers extends PerDocValues { private final TreeMap codecs = new TreeMap(); public PerDocProducers(Directory dir, FieldInfos fieldInfos, SegmentInfo si, - int readBufferSize, int indexDivisor) throws IOException { + IOContext context, int indexDivisor) throws IOException { final Map producers = new HashMap(); boolean success = false; try { @@ -229,7 +229,7 @@ final class PerFieldCodecWrapper extends Codec { Codec codec = segmentCodecs.codecs[fi.getCodecId()]; if (!producers.containsKey(codec)) { producers.put(codec, codec.docsProducer(new SegmentReadState(dir, - si, fieldInfos, readBufferSize, indexDivisor, fi.getCodecId()))); + si, fieldInfos, context, indexDivisor, fi.getCodecId()))); } codecs.put(fi.name, producers.get(codec)); } diff --git a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java index e35abf39682..68b81771fd2 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentCoreReaders.java @@ -45,7 +45,7 @@ final class SegmentCoreReaders { final Directory dir; final Directory cfsDir; - final int readBufferSize; + final IOContext context; final int termsIndexDivisor; private final SegmentReader owner; @@ -57,7 +57,7 @@ final class SegmentCoreReaders { - SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, int readBufferSize, int termsIndexDivisor) throws IOException { + SegmentCoreReaders(SegmentReader owner, Directory dir, SegmentInfo si, IOContext context, int termsIndexDivisor) throws IOException { if (termsIndexDivisor == 0) { throw new IllegalArgumentException("indexDivisor must be < 0 (don't load terms index) or greater than 0 (got 0)"); @@ -65,7 +65,7 @@ final class SegmentCoreReaders { segment = si.name; final SegmentCodecs segmentCodecs = si.getSegmentCodecs(); - this.readBufferSize = readBufferSize; + this.context = context; this.dir = dir; boolean success = false; @@ -73,7 +73,7 @@ final class SegmentCoreReaders { try { Directory dir0 = dir; if (si.getUseCompoundFile()) { - cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); dir0 = cfsReader; } cfsDir = dir0; @@ -82,7 +82,7 @@ final class SegmentCoreReaders { this.termsIndexDivisor = termsIndexDivisor; final Codec codec = segmentCodecs.codec(); - final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, readBufferSize, termsIndexDivisor); + final SegmentReadState segmentReadState = new SegmentReadState(cfsDir, si, fieldInfos, context, termsIndexDivisor); // Ask codec for its Fields fields = codec.fieldsProducer(segmentReadState); assert fields != null; @@ -163,7 +163,7 @@ final class SegmentCoreReaders { assert storeCFSReader == null; storeCFSReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.getDocStoreSegment(), "", IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), - readBufferSize); + context); storeDir = storeCFSReader; assert storeDir != null; } else { @@ -175,7 +175,7 @@ final class SegmentCoreReaders { // was not used, but then we are asked to open doc // stores after the segment has switched to CFS if (cfsReader == null) { - cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(segment, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); } storeDir = cfsReader; assert storeDir != null; @@ -185,7 +185,7 @@ final class SegmentCoreReaders { } final String storesSegment = si.getDocStoreSegment(); - fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, readBufferSize, + fieldsReaderOrig = new FieldsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); // Verify two sources of "maxDoc" agree: @@ -194,7 +194,7 @@ final class SegmentCoreReaders { } if (si.getHasVectors()) { // open term vector files only as needed - termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, readBufferSize, si.getDocStoreOffset(), si.docCount); + termVectorsReaderOrig = new TermVectorsReader(storeDir, storesSegment, fieldInfos, context, si.getDocStoreOffset(), si.docCount); } } } diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java index 0daf3e876ff..c5a572faca7 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfo.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfo.java @@ -245,7 +245,7 @@ public final class SegmentInfo implements Cloneable { } final Directory dirToTest; if (isCompoundFile) { - dirToTest = new CompoundFileReader(dir, IndexFileNames.segmentFileName(storesSegment, "", ext)); + dirToTest = new CompoundFileReader(dir, IndexFileNames.segmentFileName(storesSegment, "", ext), IOContext.READONCE); } else { dirToTest = dir; } @@ -264,7 +264,7 @@ public final class SegmentInfo implements Cloneable { Directory dir0 = dir; if (isCompoundFile && checkCompoundFile) { dir0 = new CompoundFileReader(dir, IndexFileNames.segmentFileName(name, - "", IndexFileNames.COMPOUND_FILE_EXTENSION)); + "", IndexFileNames.COMPOUND_FILE_EXTENSION), IOContext.READONCE); } try { fieldInfos = new FieldInfos(dir0, IndexFileNames.segmentFileName(name, diff --git a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java index 4a5e784da68..02f329724d3 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentInfos.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentInfos.java @@ -32,6 +32,7 @@ import java.util.Map; import java.util.Set; import org.apache.lucene.index.FieldInfos.FieldNumberBiMap; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.index.codecs.DefaultSegmentInfosWriter; import org.apache.lucene.index.codecs.SegmentInfosReader; @@ -254,7 +255,7 @@ public final class SegmentInfos implements Cloneable, Iterable { try { SegmentInfosReader infosReader = codecs.getSegmentInfosReader(); - infosReader.read(directory, segmentFileName, codecs, this); + infosReader.read(directory, segmentFileName, codecs, this, IOContext.READ); success = true; } finally { @@ -322,7 +323,7 @@ public final class SegmentInfos implements Cloneable, Iterable { try { SegmentInfosWriter infosWriter = codecs.getSegmentInfosWriter(); - segnOutput = infosWriter.writeInfos(directory, segmentFileName, this); + segnOutput = infosWriter.writeInfos(directory, segmentFileName, this, IOContext.DEFAULT); infosWriter.prepareCommit(segnOutput); pendingSegnOutput = segnOutput; success = true; @@ -597,7 +598,7 @@ public final class SegmentInfos implements Cloneable, Iterable { for(int i=0;i { } private final long writeGlobalFieldMap(FieldNumberBiMap map, Directory dir, String name) throws IOException { - final IndexOutput output = dir.createOutput(name); + final IndexOutput output = dir.createOutput(name, IOContext.READONCE); boolean success = false; long version; try { @@ -843,7 +844,7 @@ public final class SegmentInfos implements Cloneable, Iterable { private void readGlobalFieldMap(FieldNumberBiMap map, Directory dir) throws IOException { final String name = getGlobalFieldNumberName(lastGlobalFieldMapVersion); - final IndexInput input = dir.openInput(name); + final IndexInput input = dir.openInput(name, IOContext.READONCE); try { map.read(input); } finally { @@ -934,7 +935,7 @@ public final class SegmentInfos implements Cloneable, Iterable { } try { - IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN); + IndexOutput genOutput = dir.createOutput(IndexFileNames.SEGMENTS_GEN, IOContext.READONCE); try { genOutput.writeInt(FORMAT_SEGMENTS_GEN_CURRENT); genOutput.writeLong(generation); diff --git a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java index 3c098de8635..7ae6094bc01 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentMerger.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentMerger.java @@ -24,6 +24,7 @@ import java.util.Collection; import java.util.List; import org.apache.lucene.document.Document; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexReader.FieldOption; import org.apache.lucene.index.MergePolicy.MergeAbortedException; import org.apache.lucene.index.codecs.Codec; @@ -127,12 +128,12 @@ final class SegmentMerger { * deletion files, this SegmentInfo must not reference such files when this * method is called, because they are not allowed within a compound file. */ - final Collection createCompoundFile(String fileName, final SegmentInfo info) + final Collection createCompoundFile(String fileName, final SegmentInfo info, IOContext context) throws IOException { // Now merge all added files Collection files = info.files(); - CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, checkAbort); + CompoundFileWriter cfsWriter = new CompoundFileWriter(directory, fileName, context, checkAbort); for (String file : files) { assert !IndexFileNames.matchesExtension(file, IndexFileNames.DELETES_EXTENSION) : ".del file is not allowed in .cfs: " + file; @@ -232,8 +233,8 @@ final class SegmentMerger { int docCount = 0; setMatchingSegmentReaders(); - - final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment); + // nocommit - should we rather use IOContext.MERGE here? + final FieldsWriter fieldsWriter = new FieldsWriter(directory, segment, IOContext.DEFAULT); try { int idx = 0; @@ -268,8 +269,8 @@ final class SegmentMerger { // entering the index. See LUCENE-1282 for // details. throw new RuntimeException("mergeFields produced an invalid result: docCount is " + docCount + " but fdx file size is " + fdxFileLength + " file=" + fileName + " file exists?=" + directory.fileExists(fileName) + "; now aborting this merge to prevent index corruption"); - - segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null); + //nocommit if Merge then what to initialize OneMerge with ? + segmentWriteState = new SegmentWriteState(null, directory, segment, fieldInfos, docCount, termIndexInterval, codecInfo, null, IOContext.DEFAULT); return docCount; } @@ -354,8 +355,9 @@ final class SegmentMerger { * @throws IOException */ private final void mergeVectors() throws IOException { + //nocommit Putting MERGE context here would lead to assert error. What should MergeInfo be initialized with here? TermVectorsWriter termVectorsWriter = - new TermVectorsWriter(directory, segment, fieldInfos); + new TermVectorsWriter(directory, segment, fieldInfos, new IOContext(Context.DEFAULT)); try { int idx = 0; @@ -614,7 +616,8 @@ final class SegmentMerger { for (FieldInfo fi : fieldInfos) { if (fi.isIndexed && !fi.omitNorms) { if (output == null) { - output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION)); + //nocommit Putting MERGE context here would lead to assert error. What should MergeInfo be initialized with here? + output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.NORMS_EXTENSION), new IOContext(Context.DEFAULT)); output.writeBytes(SegmentNorms.NORMS_HEADER, SegmentNorms.NORMS_HEADER.length); } for (IndexReader reader : readers) { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentNorms.java b/lucene/src/java/org/apache/lucene/index/SegmentNorms.java index df8bf9a7d31..6c88ffaebe8 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentNorms.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentNorms.java @@ -20,6 +20,8 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.concurrent.atomic.AtomicInteger; +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -219,7 +221,7 @@ final class SegmentNorms implements Cloneable { // NOTE: norms are re-written in regular directory, not cfs si.advanceNormGen(this.number); final String normFileName = si.getNormFileName(this.number); - IndexOutput out = owner.directory().createOutput(normFileName); + IndexOutput out = owner.directory().createOutput(normFileName, IOContext.DEFAULT); boolean success = false; try { try { diff --git a/lucene/src/java/org/apache/lucene/index/SegmentReadState.java b/lucene/src/java/org/apache/lucene/index/SegmentReadState.java index d2159d92295..b72dc80b42c 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentReadState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentReadState.java @@ -26,7 +26,7 @@ public class SegmentReadState { public final Directory dir; public final SegmentInfo segmentInfo; public final FieldInfos fieldInfos; - public final int readBufferSize; + public final IOContext context; // NOTE: if this is < 0, that means "defer terms index // load until needed". But if the codec must load the @@ -37,20 +37,20 @@ public class SegmentReadState { public final int codecId; public SegmentReadState(Directory dir, SegmentInfo info, - FieldInfos fieldInfos, int readBufferSize, int termsIndexDivisor) { - this(dir, info, fieldInfos, readBufferSize, termsIndexDivisor, -1); + FieldInfos fieldInfos, IOContext context, int termsIndexDivisor) { + this(dir, info, fieldInfos, context, termsIndexDivisor, -1); } public SegmentReadState(Directory dir, SegmentInfo info, FieldInfos fieldInfos, - int readBufferSize, + IOContext context, int termsIndexDivisor, int codecId) { this.dir = dir; this.segmentInfo = info; this.fieldInfos = fieldInfos; - this.readBufferSize = readBufferSize; + this.context = context; this.termsIndexDivisor = termsIndexDivisor; this.codecId = codecId; } diff --git a/lucene/src/java/org/apache/lucene/index/SegmentReader.java b/lucene/src/java/org/apache/lucene/index/SegmentReader.java index 06db5f63de7..f5d36d97b00 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentReader.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentReader.java @@ -30,7 +30,6 @@ import java.util.concurrent.atomic.AtomicInteger; import org.apache.lucene.document.Document; import org.apache.lucene.document.FieldSelector; import org.apache.lucene.index.codecs.PerDocValues; -import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.util.BitVector; @@ -46,7 +45,6 @@ public class SegmentReader extends IndexReader implements Cloneable { protected boolean readOnly; private SegmentInfo si; - private int readBufferSize; private final ReaderContext readerContext = new AtomicReaderContext(this); CloseableThreadLocal fieldsReaderLocal = new FieldsReaderLocal(); CloseableThreadLocal termVectorsLocal = new CloseableThreadLocal(); @@ -88,8 +86,9 @@ public class SegmentReader extends IndexReader implements Cloneable { * @throws CorruptIndexException if the index is corrupt * @throws IOException if there is a low-level IO error */ - public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor) throws CorruptIndexException, IOException { - return get(readOnly, si.dir, si, BufferedIndexInput.BUFFER_SIZE, true, termInfosIndexDivisor); + public static SegmentReader get(boolean readOnly, SegmentInfo si, int termInfosIndexDivisor, IOContext context) throws CorruptIndexException, IOException { + // TODO should we check if readOnly and context combination makes sense like asserting that if read only we don't get a default? + return get(readOnly, si.dir, si, true, termInfosIndexDivisor, context); } /** @@ -99,25 +98,23 @@ public class SegmentReader extends IndexReader implements Cloneable { public static SegmentReader get(boolean readOnly, Directory dir, SegmentInfo si, - int readBufferSize, boolean doOpenStores, - int termInfosIndexDivisor) + int termInfosIndexDivisor, + IOContext context) throws CorruptIndexException, IOException { SegmentReader instance = new SegmentReader(); instance.readOnly = readOnly; instance.si = si; - instance.readBufferSize = readBufferSize; - boolean success = false; try { - instance.core = new SegmentCoreReaders(instance, dir, si, readBufferSize, termInfosIndexDivisor); + instance.core = new SegmentCoreReaders(instance, dir, si, context, termInfosIndexDivisor); if (doOpenStores) { instance.core.openDocStores(si); } instance.loadDeletedDocs(); - instance.openNorms(instance.core.cfsDir, readBufferSize); + instance.openNorms(instance.core.cfsDir, context); success = true; } finally { @@ -161,7 +158,7 @@ public class SegmentReader extends IndexReader implements Cloneable { private void loadDeletedDocs() throws IOException { // NOTE: the bitvector is stored using the regular directory, not cfs if (hasDeletions(si)) { - deletedDocs = new BitVector(directory(), si.getDelFileName()); + deletedDocs = new BitVector(directory(), si.getDelFileName(), IOContext.DEFAULT); deletedDocsRef = new AtomicInteger(1); assert checkDeletedCounts(); if (deletedDocs.size() != si.docCount) { @@ -250,7 +247,6 @@ public class SegmentReader extends IndexReader implements Cloneable { clone.core = core; clone.readOnly = openReadOnly; clone.si = si; - clone.readBufferSize = readBufferSize; clone.pendingDeleteCount = pendingDeleteCount; clone.readerFinishedListeners = readerFinishedListeners; @@ -295,7 +291,7 @@ public class SegmentReader extends IndexReader implements Cloneable { // If we are not cloning, then this will open anew // any norms that have changed: - clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), readBufferSize); + clone.openNorms(si.getUseCompoundFile() ? core.getCFSReader() : directory(), IOContext.DEFAULT); success = true; } finally { @@ -337,7 +333,7 @@ public class SegmentReader extends IndexReader implements Cloneable { final String delFileName = si.getDelFileName(); boolean success = false; try { - deletedDocs.write(directory(), delFileName); + deletedDocs.write(directory(), delFileName, IOContext.DEFAULT); success = true; } finally { if (!success) { @@ -576,7 +572,7 @@ public class SegmentReader extends IndexReader implements Cloneable { norm.copyOnWrite()[doc] = value; // set the value } - private void openNorms(Directory cfsDir, int readBufferSize) throws IOException { + private void openNorms(Directory cfsDir, IOContext context) throws IOException { long nextNormSeek = SegmentNorms.NORMS_HEADER.length; //skip header (header unused for now) int maxDoc = maxDoc(); for (FieldInfo fi : core.fieldInfos) { @@ -600,7 +596,7 @@ public class SegmentReader extends IndexReader implements Cloneable { if (singleNormFile) { normSeek = nextNormSeek; if (singleNormStream == null) { - singleNormStream = d.openInput(fileName, readBufferSize); + singleNormStream = d.openInput(fileName, context); singleNormRef = new AtomicInteger(1); } else { singleNormRef.incrementAndGet(); @@ -610,7 +606,7 @@ public class SegmentReader extends IndexReader implements Cloneable { // If this were to change in the future, a clone could be done here. normInput = singleNormStream; } else { - normInput = d.openInput(fileName); + normInput = d.openInput(fileName, context); // if the segment was created in 3.2 or after, we wrote the header for sure, // and don't need to do the sketchy file size check. otherwise, we check // if the size is exactly equal to maxDoc to detect a headerless file. diff --git a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java index 32fab036273..25573c62c9e 100644 --- a/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java +++ b/lucene/src/java/org/apache/lucene/index/SegmentWriteState.java @@ -51,9 +51,11 @@ public class SegmentWriteState { * slower. Searching is typically not dominated by dictionary lookup, so * tweaking this is rarely useful.*/ public int termIndexInterval; // TODO: this should be private to the codec, not settable here or in IWC + + public final IOContext context; public SegmentWriteState(PrintStream infoStream, Directory directory, String segmentName, FieldInfos fieldInfos, - int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes) { + int numDocs, int termIndexInterval, SegmentCodecs segmentCodecs, BufferedDeletes segDeletes, IOContext context) { this.infoStream = infoStream; this.segDeletes = segDeletes; this.directory = directory; @@ -63,6 +65,7 @@ public class SegmentWriteState { this.termIndexInterval = termIndexInterval; this.segmentCodecs = segmentCodecs; codecId = -1; + this.context = context; } /** @@ -76,6 +79,7 @@ public class SegmentWriteState { numDocs = state.numDocs; termIndexInterval = state.termIndexInterval; segmentCodecs = state.segmentCodecs; + context = state.context; this.codecId = codecId; segDeletes = state.segDeletes; } diff --git a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java index c3aa5c86b60..ee51d535886 100644 --- a/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/StoredFieldsWriter.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.RamUsageEstimator; @@ -77,7 +78,8 @@ final class StoredFieldsWriter { private synchronized void initFieldsWriter() throws IOException { if (fieldsWriter == null) { - fieldsWriter = new FieldsWriter(docWriter.directory, docWriter.getSegment()); + // nocommit - is this always a flush here or should we have a IOContext argument? + fieldsWriter = new FieldsWriter(docWriter.directory, docWriter.getSegment(), new IOContext(Context.FLUSH)); lastDocID = 0; } } diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java b/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java index 5c8d2e68905..bd8e9128290 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsReader.java @@ -17,6 +17,8 @@ package org.apache.lucene.index; * limitations under the License. */ +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.BufferedIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -61,29 +63,24 @@ class TermVectorsReader implements Cloneable { private final int format; - TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos) + TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context) throws CorruptIndexException, IOException { - this(d, segment, fieldInfos, BufferedIndexInput.BUFFER_SIZE); - } - - TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, int readBufferSize) - throws CorruptIndexException, IOException { - this(d, segment, fieldInfos, readBufferSize, -1, 0); + this(d, segment, fieldInfos, context, -1, 0); } - TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, int readBufferSize, int docStoreOffset, int size) + TermVectorsReader(Directory d, String segment, FieldInfos fieldInfos, IOContext context, int docStoreOffset, int size) throws CorruptIndexException, IOException { boolean success = false; try { String idxName = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION); - tvx = d.openInput(idxName, readBufferSize); + tvx = d.openInput(idxName, context); format = checkValidFormat(tvx, idxName); String fn = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION); - tvd = d.openInput(fn, readBufferSize); + tvd = d.openInput(fn, context); final int tvdFormat = checkValidFormat(tvd, fn); fn = IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION); - tvf = d.openInput(fn, readBufferSize); + tvf = d.openInput(fn, context); final int tvfFormat = checkValidFormat(tvf, fn); assert format == tvdFormat; diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java index 4fbc6beebb2..e6042910cd1 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsTermsWriter.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import java.io.IOException; import java.util.Map; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.IndexOutput; import org.apache.lucene.util.ArrayUtil; import org.apache.lucene.util.BytesRef; @@ -89,13 +90,14 @@ final class TermVectorsTermsWriter extends TermsHashConsumer { if (tvx == null) { boolean success = false; try { + IOContext context = new IOContext(Context.FLUSH); // If we hit an exception while init'ing the term // vector output files, we must abort this segment // because those files will be in an unknown // state: - tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION)); - tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)); - tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION)); + tvx = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_INDEX_EXTENSION), context); + tvd = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), context); + tvf = docWriter.directory.createOutput(IndexFileNames.segmentFileName(docWriter.getSegment(), "", IndexFileNames.VECTORS_FIELDS_EXTENSION), context); tvx.writeInt(TermVectorsReader.FORMAT_CURRENT); tvd.writeInt(TermVectorsReader.FORMAT_CURRENT); diff --git a/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java b/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java index 44d1b5b4391..1bc34b94521 100644 --- a/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/TermVectorsWriter.java @@ -31,15 +31,15 @@ final class TermVectorsWriter { private FieldInfos fieldInfos; public TermVectorsWriter(Directory directory, String segment, - FieldInfos fieldInfos) throws IOException { + FieldInfos fieldInfos, IOContext context) throws IOException { boolean success = false; try { // Open files for TermVector storage - tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION)); + tvx = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_INDEX_EXTENSION), context); tvx.writeInt(TermVectorsReader.FORMAT_CURRENT); - tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION)); + tvd = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_DOCUMENTS_EXTENSION), context); tvd.writeInt(TermVectorsReader.FORMAT_CURRENT); - tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION)); + tvf = directory.createOutput(IndexFileNames.segmentFileName(segment, "", IndexFileNames.VECTORS_FIELDS_EXTENSION), context); tvf.writeInt(TermVectorsReader.FORMAT_CURRENT); success = true; } finally { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java index 1552f4fe24f..592539b4ec9 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java @@ -29,6 +29,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; @@ -107,7 +108,7 @@ public class BlockTermsReader extends FieldsProducer { //private String segment; - public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, int readBufferSize, + public BlockTermsReader(TermsIndexReaderBase indexReader, Directory dir, FieldInfos fieldInfos, String segment, PostingsReaderBase postingsReader, IOContext context, int termsCacheSize, int codecId) throws IOException { @@ -116,7 +117,7 @@ public class BlockTermsReader extends FieldsProducer { //this.segment = segment; in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, BlockTermsWriter.TERMS_EXTENSION), - readBufferSize); + context); boolean success = false; try { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java index 9cb9d4cbd8d..926a6af6301 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsWriter.java @@ -72,7 +72,7 @@ public class BlockTermsWriter extends FieldsConsumer { throws IOException { final String termsFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_EXTENSION); this.termsIndexWriter = termsIndexWriter; - out = state.directory.createOutput(termsFileName); + out = state.directory.createOutput(termsFileName, state.context); boolean success = false; try { fieldInfos = state.fieldInfos; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java index c00f54fbbd2..7c6091aacdc 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultDocValuesProducer.java @@ -22,6 +22,7 @@ import java.util.TreeMap; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.values.Bytes; import org.apache.lucene.index.values.IndexDocValues; @@ -57,6 +58,7 @@ public class DefaultDocValuesProducer extends PerDocValues { */ public DefaultDocValuesProducer(SegmentInfo si, Directory dir, FieldInfos fieldInfo, int codecId) throws IOException { + //nocommit this needs an IOContext docValues = load(fieldInfo, si.name, si.docCount, dir, codecId); } @@ -120,6 +122,7 @@ public class DefaultDocValuesProducer extends PerDocValues { */ protected IndexDocValues loadDocValues(int docCount, Directory dir, String id, ValueType type) throws IOException { + // nocommit this needs an IOContext too switch (type) { case INTS: return Ints.getValues(dir, id, false); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java index f5153b368a7..6a5eb120c9d 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosReader.java @@ -22,11 +22,13 @@ import java.io.IOException; import org.apache.lucene.index.CompoundFileReader; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldsReader; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.IndexFormatTooOldException; import org.apache.lucene.index.IndexFormatTooNewException; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.ChecksumIndexInput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -39,10 +41,10 @@ public class DefaultSegmentInfosReader extends SegmentInfosReader { @Override public void read(Directory directory, String segmentsFileName, CodecProvider codecs, - SegmentInfos infos) throws IOException { + SegmentInfos infos, IOContext context) throws IOException { IndexInput input = null; try { - input = openInput(directory, segmentsFileName); + input = openInput(directory, segmentsFileName, context); final int format = input.readInt(); infos.setFormat(format); @@ -70,11 +72,11 @@ public class DefaultSegmentInfosReader extends SegmentInfosReader { if (si.getDocStoreIsCompoundFile()) { dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( si.getDocStoreSegment(), "", - IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), 1024); + IndexFileNames.COMPOUND_FILE_STORE_EXTENSION), context); } } else if (si.getUseCompoundFile()) { dir = new CompoundFileReader(dir, IndexFileNames.segmentFileName( - si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), 1024); + si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); } try { @@ -108,8 +110,8 @@ public class DefaultSegmentInfosReader extends SegmentInfosReader { } - public IndexInput openInput(Directory dir, String segmentsFileName) throws IOException { - IndexInput in = dir.openInput(segmentsFileName); + public IndexInput openInput(Directory dir, String segmentsFileName, IOContext context) throws IOException { + IndexInput in = dir.openInput(segmentsFileName, context); return new ChecksumIndexInput(in); } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java index 7a1b61b597f..dca66bd94ec 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java @@ -19,8 +19,10 @@ package org.apache.lucene.index.codecs; import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentInfos; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.ChecksumIndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -54,9 +56,10 @@ public class DefaultSegmentInfosWriter extends SegmentInfosWriter { public static final int FORMAT_MINIMUM = FORMAT_DIAGNOSTICS; @Override - public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos) + public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context) throws IOException { - IndexOutput out = createOutput(dir, segmentFileName); + //nocommit should this context always be flush? + IndexOutput out = createOutput(dir, segmentFileName, context); boolean success = false; try { out.writeInt(FORMAT_CURRENT); // write FORMAT @@ -77,9 +80,9 @@ public class DefaultSegmentInfosWriter extends SegmentInfosWriter { } } - protected IndexOutput createOutput(Directory dir, String segmentFileName) + protected IndexOutput createOutput(Directory dir, String segmentFileName, IOContext context) throws IOException { - IndexOutput plainOut = dir.createOutput(segmentFileName); + IndexOutput plainOut = dir.createOutput(segmentFileName, context); ChecksumIndexOutput out = new ChecksumIndexOutput(plainOut); return out; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java index ad48f039020..fa2fefacad7 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexReader.java @@ -21,6 +21,8 @@ import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.util.BytesRef; import org.apache.lucene.util.CodecUtil; @@ -68,12 +70,12 @@ public class FixedGapTermsIndexReader extends TermsIndexReaderBase { // start of the field info data protected long dirOffset; - public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId) + public FixedGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, Comparator termComp, int codecId, IOContext context) throws IOException { this.termComp = termComp; - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, FixedGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java index 38f094d47c0..28149ee2533 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/FixedGapTermsIndexWriter.java @@ -58,7 +58,7 @@ public class FixedGapTermsIndexWriter extends TermsIndexWriterBase { public FixedGapTermsIndexWriter(SegmentWriteState state) throws IOException { final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); termIndexInterval = state.termIndexInterval; - out = state.directory.createOutput(indexFileName); + out = state.directory.createOutput(indexFileName, state.context); boolean success = false; try { fieldInfos = state.fieldInfos; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java index 4a90fb93ac2..6abd7f6b4a1 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosReader.java @@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs; import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.Directory; @@ -36,5 +37,5 @@ public abstract class SegmentInfosReader { * @param infos empty instance to be populated with data * @throws IOException */ - public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos) throws IOException; + public abstract void read(Directory directory, String segmentsFileName, CodecProvider codecs, SegmentInfos infos, IOContext context) throws IOException; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java index 19f2e5dc397..84b2248aaa4 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/SegmentInfosWriter.java @@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs; import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.SegmentInfos; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -41,7 +42,7 @@ public abstract class SegmentInfosWriter { * phase commit" operations as described above. * @throws IOException */ - public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos) throws IOException; + public abstract IndexOutput writeInfos(Directory dir, String segmentsFileName, SegmentInfos infos, IOContext context) throws IOException; /** * First phase of the two-phase commit - ensure that all output can be diff --git a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java index e66f413d54c..065fe4f7e8b 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexReader.java @@ -27,6 +27,8 @@ import java.util.Iterator; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.store.Directory; @@ -57,10 +59,10 @@ public class VariableGapTermsIndexReader extends TermsIndexReaderBase { protected long dirOffset; final String segment; - public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId) + public VariableGapTermsIndexReader(Directory dir, FieldInfos fieldInfos, String segment, int indexDivisor, int codecId, IOContext context) throws IOException { - in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION)); + in = dir.openInput(IndexFileNames.segmentFileName(segment, codecId, VariableGapTermsIndexWriter.TERMS_INDEX_EXTENSION), context); this.segment = segment; boolean success = false; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java index d10608879cb..c8d3d3e32ca 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/VariableGapTermsIndexWriter.java @@ -159,7 +159,7 @@ public class VariableGapTermsIndexWriter extends TermsIndexWriterBase { public VariableGapTermsIndexWriter(SegmentWriteState state, IndexTermSelector policy) throws IOException { final String indexFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, TERMS_INDEX_EXTENSION); - out = state.directory.createOutput(indexFileName); + out = state.directory.createOutput(indexFileName, state.context); boolean success = false; try { fieldInfos = state.fieldInfos; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java index 8b103cf21b3..48594ca1aa0 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryCodec.java @@ -242,7 +242,7 @@ public class MemoryCodec extends Codec { public FieldsConsumer fieldsConsumer(SegmentWriteState state) throws IOException { final String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, EXTENSION); - final IndexOutput out = state.directory.createOutput(fileName); + final IndexOutput out = state.directory.createOutput(fileName, state.context); return new FieldsConsumer() { @Override @@ -697,8 +697,9 @@ public class MemoryCodec extends Codec { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { + //nocommit its seems due to the nature of this codec that we should use IOContext.READONCE here where applicable. final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, EXTENSION); - final IndexInput in = state.dir.openInput(fileName); + final IndexInput in = state.dir.openInput(fileName, state.context); final SortedMap fields = new TreeMap(); diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java index 763457fa71f..087be5e527d 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexCodec.java @@ -65,7 +65,7 @@ public class PreFlexCodec extends Codec { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.readBufferSize, state.termsIndexDivisor); + return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java index 025b0bf840f..7d5f0ee4db6 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/PreFlexFields.java @@ -31,6 +31,7 @@ import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; import org.apache.lucene.index.FieldInfos; import org.apache.lucene.index.FieldsEnum; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.Term; @@ -62,10 +63,10 @@ public class PreFlexFields extends FieldsProducer { final TreeMap fields = new TreeMap(); final Map preTerms = new HashMap(); private final Directory dir; - private final int readBufferSize; + private final IOContext context; private Directory cfsReader; - public PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, int readBufferSize, int indexDivisor) + public PreFlexFields(Directory dir, FieldInfos fieldInfos, SegmentInfo info, IOContext context, int indexDivisor) throws IOException { si = info; @@ -80,19 +81,19 @@ public class PreFlexFields extends FieldsProducer { boolean success = false; try { - TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, readBufferSize, indexDivisor); + TermInfosReader r = new TermInfosReader(dir, info.name, fieldInfos, context, indexDivisor); if (indexDivisor == -1) { tisNoIndex = r; } else { tisNoIndex = null; tis = r; } - this.readBufferSize = readBufferSize; + this.context = context; this.fieldInfos = fieldInfos; // make sure that all index files have been read or are kept open // so that if an index update removes them we'll still have them - freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION), readBufferSize); + freqStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.FREQ_EXTENSION), context); boolean anyProx = false; for (FieldInfo fi : fieldInfos) { if (fi.isIndexed) { @@ -105,7 +106,7 @@ public class PreFlexFields extends FieldsProducer { } if (anyProx) { - proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION), readBufferSize); + proxStream = dir.openInput(IndexFileNames.segmentFileName(info.name, "", PreFlexCodec.PROX_EXTENSION), context); } else { proxStream = null; } @@ -178,7 +179,7 @@ public class PreFlexFields extends FieldsProducer { // to CFS if (!(dir instanceof CompoundFileReader)) { - dir0 = cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), readBufferSize); + dir0 = cfsReader = new CompoundFileReader(dir, IndexFileNames.segmentFileName(si.name, "", IndexFileNames.COMPOUND_FILE_EXTENSION), context); } else { dir0 = dir; } @@ -187,7 +188,7 @@ public class PreFlexFields extends FieldsProducer { dir0 = dir; } - tis = new TermInfosReader(dir0, si.name, fieldInfos, readBufferSize, indexDivisor); + tis = new TermInfosReader(dir0, si.name, fieldInfos, context, indexDivisor); } } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java index 8178bd33cf6..34f2f62e816 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/preflex/TermInfosReader.java @@ -21,6 +21,7 @@ import java.io.IOException; import org.apache.lucene.index.CorruptIndexException; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.Directory; @@ -94,7 +95,7 @@ public final class TermInfosReader { SegmentTermEnum termEnum; } - TermInfosReader(Directory dir, String seg, FieldInfos fis, int readBufferSize, int indexDivisor) + TermInfosReader(Directory dir, String seg, FieldInfos fis, IOContext context, int indexDivisor) throws CorruptIndexException, IOException { boolean success = false; @@ -108,7 +109,7 @@ public final class TermInfosReader { fieldInfos = fis; origEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_EXTENSION), - readBufferSize), fieldInfos, false); + context), fieldInfos, false); size = origEnum.size; @@ -116,7 +117,7 @@ public final class TermInfosReader { // Load terms index totalIndexInterval = origEnum.indexInterval * indexDivisor; final SegmentTermEnum indexEnum = new SegmentTermEnum(directory.openInput(IndexFileNames.segmentFileName(segment, "", PreFlexCodec.TERMS_INDEX_EXTENSION), - readBufferSize), fieldInfos, true); + context), fieldInfos, true); try { int indexSize = 1+((int)indexEnum.size-1)/indexDivisor; // otherwise read index diff --git a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java index 3f88575b92f..157e8c6c448 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingCodec.java @@ -110,7 +110,7 @@ public class PulsingCodec extends Codec { // We wrap StandardPostingsReader, but any StandardPostingsReader // will work: - PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); + PostingsReaderBase docsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); PostingsReaderBase pulsingReader = new PulsingPostingsReaderImpl(docsReader); // Terms dict index reader @@ -122,7 +122,7 @@ public class PulsingCodec extends Codec { state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - state.codecId); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -136,7 +136,7 @@ public class PulsingCodec extends Codec { FieldsProducer ret = new BlockTermsReader(indexReader, state.dir, state.fieldInfos, state.segmentInfo.name, pulsingReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java index da91f2b4c5e..2bf3c5f4330 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/IntStreamFactory.java @@ -17,6 +17,7 @@ package org.apache.lucene.index.codecs.sep; * limitations under the License. */ +import org.apache.lucene.index.IOContext; import org.apache.lucene.store.Directory; import org.apache.lucene.store.BufferedIndexInput; @@ -24,10 +25,10 @@ import java.io.IOException; /** @lucene.experimental */ public abstract class IntStreamFactory { - public IntIndexInput openInput(Directory dir, String fileName) throws IOException { - return openInput(dir, fileName, BufferedIndexInput.BUFFER_SIZE); + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return openInput(dir, fileName, context); } - public abstract IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException; +// public abstract IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException; public abstract IntIndexOutput createOutput(Directory dir, String fileName) throws IOException; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java index a6588d5c217..82ec9467e0f 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReaderImpl.java @@ -23,6 +23,7 @@ import java.util.Collection; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; @@ -58,19 +59,19 @@ public class SepPostingsReaderImpl extends PostingsReaderBase { int maxSkipLevels; int skipMinimum; - public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, int readBufferSize, IntStreamFactory intFactory, int codecId) throws IOException { + public SepPostingsReaderImpl(Directory dir, SegmentInfo segmentInfo, IOContext context, IntStreamFactory intFactory, int codecId) throws IOException { boolean success = false; try { final String docFileName = IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.DOC_EXTENSION); - docIn = intFactory.openInput(dir, docFileName); + docIn = intFactory.openInput(dir, docFileName, context); - skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION), readBufferSize); + skipIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.SKIP_EXTENSION), context); if (segmentInfo.getHasProx()) { - freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.FREQ_EXTENSION)); - posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION), readBufferSize); - payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION), readBufferSize); + freqIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.FREQ_EXTENSION), context); + posIn = intFactory.openInput(dir, IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.POS_EXTENSION), context); + payloadIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, SepPostingsWriterImpl.PAYLOAD_EXTENSION), context); } else { posIn = null; payloadIn = null; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java index ac72d5a0125..637b88f33c3 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsWriterImpl.java @@ -132,11 +132,11 @@ public final class SepPostingsWriterImpl extends PostingsWriterBase { // TODO: -- only if at least one field stores payloads? final String payloadFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, PAYLOAD_EXTENSION); - payloadOut = state.directory.createOutput(payloadFileName); + payloadOut = state.directory.createOutput(payloadFileName, state.context); } final String skipFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SKIP_EXTENSION); - skipOut = state.directory.createOutput(skipFileName); + skipOut = state.directory.createOutput(skipFileName, state.context); totalNumDocs = state.numDocs; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java index e92dfbc82b5..5bcff12cacb 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java @@ -58,7 +58,7 @@ class SimpleTextFieldsReader extends FieldsProducer { final static BytesRef PAYLOAD = SimpleTextFieldsWriter.PAYLOAD; public SimpleTextFieldsReader(SegmentReadState state) throws IOException { - in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId)); + in = state.dir.openInput(SimpleTextCodec.getPostingsFileName(state.segmentInfo.name, state.codecId), state.context); fieldInfos = state.fieldInfos; } diff --git a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java index d1d5f333a50..f822ec6a72c 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsWriter.java @@ -46,7 +46,7 @@ class SimpleTextFieldsWriter extends FieldsConsumer { public SimpleTextFieldsWriter(SegmentWriteState state) throws IOException { final String fileName = SimpleTextCodec.getPostingsFileName(state.segmentName, state.codecId); - out = state.directory.createOutput(fileName); + out = state.directory.createOutput(fileName, state.context); } private void write(String s) throws IOException { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java index 9bd8bd58de6..10014389dd3 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardCodec.java @@ -89,7 +89,7 @@ public class StandardCodec extends Codec { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); + PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -98,7 +98,7 @@ public class StandardCodec extends Codec { state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - state.codecId); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -113,7 +113,7 @@ public class StandardCodec extends Codec { state.fieldInfos, state.segmentInfo.name, postings, - state.readBufferSize, + state.context, TERMS_CACHE_SIZE, state.codecId); success = true; diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java index 3ef44dd33c0..5e1d5296677 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsReader.java @@ -23,6 +23,7 @@ import java.util.Collection; import org.apache.lucene.index.DocsAndPositionsEnum; import org.apache.lucene.index.DocsEnum; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.TermState; @@ -51,15 +52,15 @@ public class StandardPostingsReader extends PostingsReaderBase { //private String segment; - public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, int readBufferSize, int codecId) throws IOException { + public StandardPostingsReader(Directory dir, SegmentInfo segmentInfo, IOContext context, int codecId) throws IOException { freqIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.FREQ_EXTENSION), - readBufferSize); + context); //this.segment = segmentInfo.name; if (segmentInfo.getHasProx()) { boolean success = false; try { proxIn = dir.openInput(IndexFileNames.segmentFileName(segmentInfo.name, codecId, StandardCodec.PROX_EXTENSION), - readBufferSize); + context); success = true; } finally { if (!success) { diff --git a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java index 474485be200..0549cc6fb04 100644 --- a/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java +++ b/lucene/src/java/org/apache/lucene/index/codecs/standard/StandardPostingsWriter.java @@ -92,14 +92,14 @@ public final class StandardPostingsWriter extends PostingsWriterBase { this.skipMinimum = skipInterval; /* set to the same for now */ //this.segment = state.segmentName; String fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.FREQ_EXTENSION); - freqOut = state.directory.createOutput(fileName); + freqOut = state.directory.createOutput(fileName, state.context); boolean success = false; try { if (state.fieldInfos.hasProx()) { // At least one field does not omit TF, so create the // prox file fileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, StandardCodec.PROX_EXTENSION); - proxOut = state.directory.createOutput(fileName); + proxOut = state.directory.createOutput(fileName, state.context); } else { // Every field omits TF so we will write no prox file proxOut = null; diff --git a/lucene/src/java/org/apache/lucene/index/values/Bytes.java b/lucene/src/java/org/apache/lucene/index/values/Bytes.java index 0944fc9ee4c..8742b0c6900 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Bytes.java +++ b/lucene/src/java/org/apache/lucene/index/values/Bytes.java @@ -23,6 +23,7 @@ import java.util.Collection; import java.util.Comparator; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.values.IndexDocValues.SortedSource; import org.apache.lucene.index.values.IndexDocValues.Source; @@ -108,6 +109,8 @@ public final class Bytes { public static Writer getWriter(Directory dir, String id, Mode mode, Comparator comp, boolean fixedSize, AtomicLong bytesUsed) throws IOException { + //nocommit this and all the blow need an IOContext too + // TODO -- i shouldn't have to specify fixed? can // track itself & do the write thing at write time? if (comp == null) { @@ -158,6 +161,8 @@ public final class Bytes { */ public static IndexDocValues getValues(Directory dir, String id, Mode mode, boolean fixedSize, int maxDoc) throws IOException { + //nocommit this and all the readers below need an IOContext too + // TODO -- I can peek @ header to determing fixed/mode? if (fixedSize) { if (mode == Mode.STRAIGHT) { @@ -344,18 +349,18 @@ public final class Bytes { protected BytesWriterBase(Directory dir, String id, String codecName, int version, boolean initIndex, ByteBlockPool pool, - AtomicLong bytesUsed) throws IOException { + AtomicLong bytesUsed, IOContext context) throws IOException { super(bytesUsed); this.id = id; this.pool = pool; datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - DATA_EXTENSION)); + DATA_EXTENSION), context); boolean success = false; try { CodecUtil.writeHeader(datOut, codecName, version); if (initIndex) { idxOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - INDEX_EXTENSION)); + INDEX_EXTENSION), context); CodecUtil.writeHeader(idxOut, codecName, version); } else { idxOut = null; @@ -428,16 +433,16 @@ public final class Bytes { protected final String id; protected BytesReaderBase(Directory dir, String id, String codecName, - int maxVersion, boolean doIndex) throws IOException { + int maxVersion, boolean doIndex, IOContext context) throws IOException { this.id = id; datIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); boolean success = false; try { version = CodecUtil.checkHeader(datIn, codecName, maxVersion, maxVersion); if (doIndex) { idxIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.INDEX_EXTENSION)); + Writer.INDEX_EXTENSION), context); final int version2 = CodecUtil.checkHeader(idxIn, codecName, maxVersion, maxVersion); assert version == version2; diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java index 56493a20e63..d6de9a053e6 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedDerefBytesImpl.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.values; import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.values.Bytes.BytesBaseSource; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; @@ -56,6 +57,7 @@ class FixedDerefBytesImpl { BytesRefHash.DEFAULT_CAPACITY, bytesUsed)); public Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException { + //nocommit this needs an IOContext too this(dir, id, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed); } @@ -63,7 +65,7 @@ class FixedDerefBytesImpl { public Writer(Directory dir, String id, Allocator allocator, AtomicLong bytesUsed) throws IOException { super(dir, id, CODEC_NAME, VERSION_CURRENT, true, - new ByteBlockPool(allocator), bytesUsed); + new ByteBlockPool(allocator), bytesUsed, IOContext.DEFAULT); docToID = new int[1]; bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT); // TODO BytesRefHash // uses bytes too! @@ -134,7 +136,7 @@ class FixedDerefBytesImpl { private final int size; Reader(Directory dir, String id, int maxDoc) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, true); + super(dir, id, CODEC_NAME, VERSION_START, true, IOContext.DEFAULT); size = datIn.readInt(); } diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java index 3a32f9892c3..e53536caf4e 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedSortedBytesImpl.java @@ -21,6 +21,7 @@ import java.io.IOException; import java.util.Comparator; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.values.Bytes.BytesBaseSortedSource; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; @@ -63,6 +64,7 @@ class FixedSortedBytesImpl { public Writer(Directory dir, String id, Comparator comp, AtomicLong bytesUsed) throws IOException { + //nocommit this needs an IOContext too this(dir, id, comp, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed); } @@ -70,7 +72,7 @@ class FixedSortedBytesImpl { public Writer(Directory dir, String id, Comparator comp, Allocator allocator, AtomicLong bytesUsed) throws IOException { super(dir, id, CODEC_NAME, VERSION_CURRENT, true, - new ByteBlockPool(allocator), bytesUsed); + new ByteBlockPool(allocator), bytesUsed, IOContext.DEFAULT); docToEntry = new int[1]; // docToEntry[0] = -1; bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT); @@ -161,7 +163,7 @@ class FixedSortedBytesImpl { private final int size; public Reader(Directory dir, String id, int maxDoc) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, true); + super(dir, id, CODEC_NAME, VERSION_START, true, IOContext.DEFAULT); size = datIn.readInt(); } diff --git a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java index d24a83a56a5..be5b342f863 100644 --- a/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/FixedStraightBytesImpl.java @@ -19,6 +19,7 @@ package org.apache.lucene.index.values; import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.values.Bytes.BytesBaseSource; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; @@ -47,7 +48,8 @@ class FixedStraightBytesImpl { private byte[] oneRecord; public Writer(Directory dir, String id) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, false, null, null); + //nocommit this needs an IOContext too + super(dir, id, CODEC_NAME, VERSION_CURRENT, false, null, null, IOContext.DEFAULT); } @@ -126,7 +128,7 @@ class FixedStraightBytesImpl { private final int maxDoc; Reader(Directory dir, String id, int maxDoc) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, false); + super(dir, id, CODEC_NAME, VERSION_START, false, IOContext.DEFAULT); size = datIn.readInt(); this.maxDoc = maxDoc; } diff --git a/lucene/src/java/org/apache/lucene/index/values/Floats.java b/lucene/src/java/org/apache/lucene/index/values/Floats.java index e4200b6ab3d..5f2f9743931 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Floats.java +++ b/lucene/src/java/org/apache/lucene/index/values/Floats.java @@ -20,6 +20,8 @@ import java.io.IOException; import java.util.Collection; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.values.IndexDocValues.Source; import org.apache.lucene.store.Directory; @@ -65,7 +67,8 @@ public class Floats { public static IndexDocValues getValues(Directory dir, String id, int maxDoc) throws IOException { - return new FloatsReader(dir, id, maxDoc); + //nocommit this needs an IOContext too + return new FloatsReader(dir, id, maxDoc, IOContext.READ); } abstract static class FloatsWriter extends Writer { @@ -76,12 +79,12 @@ public class Floats { private final byte precision; protected FloatsWriter(Directory dir, String id, int precision, - AtomicLong bytesUsed) throws IOException { + AtomicLong bytesUsed, IOContext context) throws IOException { super(bytesUsed); this.id = id; this.precision = (byte) precision; datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); boolean success = false; try { CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT); @@ -146,7 +149,7 @@ public class Floats { protected Float4Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException { - super(dir, id, 4, bytesUsed); + super(dir, id, 4, bytesUsed, new IOContext(Context.FLUSH)); } @Override @@ -189,7 +192,7 @@ public class Floats { protected Float8Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException { - super(dir, id, 8, bytesUsed); + super(dir, id, 8, bytesUsed, new IOContext(Context.FLUSH)); } @Override @@ -237,10 +240,10 @@ public class Floats { // TODO(simonw) is ByteBuffer the way to go here? private final int maxDoc; - protected FloatsReader(Directory dir, String id, int maxDoc) + protected FloatsReader(Directory dir, String id, int maxDoc, IOContext context) throws IOException { datIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START); precisionBytes = datIn.readByte(); assert precisionBytes == 4 || precisionBytes == 8; diff --git a/lucene/src/java/org/apache/lucene/index/values/Ints.java b/lucene/src/java/org/apache/lucene/index/values/Ints.java index d3cf1039538..7f81fc7d301 100644 --- a/lucene/src/java/org/apache/lucene/index/values/Ints.java +++ b/lucene/src/java/org/apache/lucene/index/values/Ints.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.values; import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.values.IntsImpl.IntsReader; import org.apache.lucene.index.values.IntsImpl.IntsWriter; import org.apache.lucene.store.Directory; @@ -36,11 +37,12 @@ public class Ints { public static Writer getWriter(Directory dir, String id, boolean useFixedArray, AtomicLong bytesUsed) throws IOException { // TODO - implement fixed?! - return new IntsWriter(dir, id, bytesUsed); + return new IntsWriter(dir, id, bytesUsed, IOContext.DEFAULT); } public static IndexDocValues getValues(Directory dir, String id, boolean useFixedArray) throws IOException { - return new IntsReader(dir, id); + //nocommit this needs an IOContext too + return new IntsReader(dir, id, IOContext.DEFAULT); } } diff --git a/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java b/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java index dc626e6fa84..9660a62c704 100644 --- a/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/IntsImpl.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.util.Collection; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; @@ -58,11 +59,11 @@ class IntsImpl { private int lastDocId = -1; private IndexOutput datOut; - protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed) + protected IntsWriter(Directory dir, String id, AtomicLong bytesUsed, IOContext context) throws IOException { super(bytesUsed); datOut = dir.createOutput(IndexFileNames.segmentFileName(id, "", - DATA_EXTENSION)); + DATA_EXTENSION), context); boolean success = false; try { CodecUtil.writeHeader(datOut, CODEC_NAME, VERSION_CURRENT); @@ -185,9 +186,9 @@ class IntsImpl { private final IndexInput datIn; private final boolean packed; - protected IntsReader(Directory dir, String id) throws IOException { + protected IntsReader(Directory dir, String id, IOContext context) throws IOException { datIn = dir.openInput(IndexFileNames.segmentFileName(id, "", - Writer.DATA_EXTENSION)); + Writer.DATA_EXTENSION), context); boolean success = false; try { CodecUtil.checkHeader(datIn, CODEC_NAME, VERSION_START, VERSION_START); diff --git a/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java index 215acd469ea..696ca01367a 100644 --- a/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/VarDerefBytesImpl.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.values; import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.values.Bytes.BytesBaseSource; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; @@ -117,6 +118,7 @@ class VarDerefBytesImpl { public Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException { + //nocommit this needs an IOContext too this(dir, id, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed); } @@ -124,7 +126,7 @@ class VarDerefBytesImpl { public Writer(Directory dir, String id, Allocator allocator, AtomicLong bytesUsed) throws IOException { super(dir, id, CODEC_NAME, VERSION_CURRENT, true, - new ByteBlockPool(allocator), bytesUsed); + new ByteBlockPool(allocator), bytesUsed, IOContext.DEFAULT); docToAddress = new int[1]; bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT); } @@ -202,7 +204,7 @@ class VarDerefBytesImpl { public static class Reader extends BytesReaderBase { Reader(Directory dir, String id, int maxDoc) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, true); + super(dir, id, CODEC_NAME, VERSION_START, true, IOContext.DEFAULT); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java index 89d4b7b1bf6..e0cfd26b504 100644 --- a/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/VarSortedBytesImpl.java @@ -22,6 +22,7 @@ import java.util.Arrays; import java.util.Comparator; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.values.Bytes.BytesBaseSortedSource; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; @@ -62,6 +63,7 @@ class VarSortedBytesImpl { public Writer(Directory dir, String id, Comparator comp, AtomicLong bytesUsed) throws IOException { + //nocommit this needs an IOContext too this(dir, id, comp, new DirectTrackingAllocator(ByteBlockPool.BYTE_BLOCK_SIZE, bytesUsed), bytesUsed); } @@ -69,7 +71,7 @@ class VarSortedBytesImpl { public Writer(Directory dir, String id, Comparator comp, Allocator allocator, AtomicLong bytesUsed) throws IOException { super(dir, id, CODEC_NAME, VERSION_CURRENT, true, - new ByteBlockPool(allocator), bytesUsed); + new ByteBlockPool(allocator), bytesUsed, IOContext.DEFAULT); this.comp = comp; docToEntry = new int[1]; docToEntry[0] = -1; @@ -157,7 +159,7 @@ class VarSortedBytesImpl { public static class Reader extends BytesReaderBase { Reader(Directory dir, String id, int maxDoc) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, true); + super(dir, id, CODEC_NAME, VERSION_START, true, IOContext.DEFAULT); } @Override diff --git a/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java b/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java index 8d0bb19c04e..82def807ede 100644 --- a/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java +++ b/lucene/src/java/org/apache/lucene/index/values/VarStraightBytesImpl.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.values; import java.io.IOException; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.values.Bytes.BytesBaseSource; import org.apache.lucene.index.values.Bytes.BytesReaderBase; import org.apache.lucene.index.values.Bytes.BytesWriterBase; @@ -51,7 +52,8 @@ class VarStraightBytesImpl { public Writer(Directory dir, String id, AtomicLong bytesUsed) throws IOException { - super(dir, id, CODEC_NAME, VERSION_CURRENT, true, null, bytesUsed); + //nocommit this needs an IOContext too + super(dir, id, CODEC_NAME, VERSION_CURRENT, true, null, bytesUsed, IOContext.DEFAULT); docToAddress = new long[1]; bytesUsed.addAndGet(RamUsageEstimator.NUM_BYTES_INT); } @@ -122,7 +124,7 @@ class VarStraightBytesImpl { private final int maxDoc; Reader(Directory dir, String id, int maxDoc) throws IOException { - super(dir, id, CODEC_NAME, VERSION_START, true); + super(dir, id, CODEC_NAME, VERSION_START, true, IOContext.DEFAULT); this.maxDoc = maxDoc; } diff --git a/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java b/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java index d8ed2c771fc..9d45492d717 100644 --- a/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java +++ b/lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java @@ -43,8 +43,9 @@ public abstract class BufferedIndexInput extends IndexInput { public BufferedIndexInput() {} /** Inits BufferedIndexInput with a specific bufferSize */ + //nocommit To cleanup class variable bufferSize as the the default size is always used public BufferedIndexInput(int bufferSize) { - checkBufferSize(bufferSize); + checkBufferSize(BufferedIndexInput.BUFFER_SIZE); this.bufferSize = bufferSize; } diff --git a/lucene/src/java/org/apache/lucene/store/Directory.java b/lucene/src/java/org/apache/lucene/store/Directory.java index 2aba2e7ef12..c30aa26286d 100644 --- a/lucene/src/java/org/apache/lucene/store/Directory.java +++ b/lucene/src/java/org/apache/lucene/store/Directory.java @@ -22,6 +22,7 @@ import java.io.IOException; import java.io.Closeable; import java.util.Collection; // for javadocs +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.IOUtils; /** A Directory is a flat list of files. Files may be written once, when they @@ -87,7 +88,7 @@ public abstract class Directory implements Closeable { /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */ - public abstract IndexOutput createOutput(String name) + public abstract IndexOutput createOutput(String name, IOContext context) throws IOException; /** @@ -103,10 +104,6 @@ public abstract class Directory implements Closeable { */ public abstract void sync(Collection names) throws IOException; - /** Returns a stream reading an existing file. */ - public abstract IndexInput openInput(String name) - throws IOException; - /** Returns a stream reading an existing file, with the * specified read buffer size. The particular Directory * implementation may ignore the buffer size. Currently @@ -114,9 +111,7 @@ public abstract class Directory implements Closeable { * parameter are {@link FSDirectory} and {@link * org.apache.lucene.index.CompoundFileReader}. */ - public IndexInput openInput(String name, int bufferSize) throws IOException { - return openInput(name); - } + public abstract IndexInput openInput(String name, IOContext context) throws IOException; /** Construct a {@link Lock}. * @param name the name of the lock file @@ -199,9 +194,9 @@ public abstract class Directory implements Closeable { * NOTE: this method does not check whether dest exist and will * overwrite it if it does. */ - public void copy(Directory to, String src, String dest) throws IOException { - IndexOutput os = to.createOutput(dest); - IndexInput is = openInput(src); + public void copy(Directory to, String src, String dest, IOContext context) throws IOException { + IndexOutput os = to.createOutput(dest, context); + IndexInput is = openInput(src, context); IOException priorException = null; try { is.copyBytes(os, is.length()); diff --git a/lucene/src/java/org/apache/lucene/store/FSDirectory.java b/lucene/src/java/org/apache/lucene/store/FSDirectory.java index 64989542da2..e546d9263f0 100644 --- a/lucene/src/java/org/apache/lucene/store/FSDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/FSDirectory.java @@ -29,6 +29,7 @@ import java.util.HashSet; import java.util.Set; import java.util.concurrent.Future; +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.ThreadInterruptedException; import org.apache.lucene.util.Constants; @@ -286,7 +287,7 @@ public abstract class FSDirectory extends Directory { /** Creates an IndexOutput for the file with the given name. */ @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); ensureCanWrite(name); @@ -319,13 +320,6 @@ public abstract class FSDirectory extends Directory { staleFiles.removeAll(toSync); } - // Inherit javadoc - @Override - public IndexInput openInput(String name) throws IOException { - ensureOpen(); - return openInput(name, BufferedIndexInput.BUFFER_SIZE); - } - @Override public String getLockID() { ensureOpen(); diff --git a/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java b/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java index 495ea02b208..dcd9162c4a2 100644 --- a/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/FileSwitchDirectory.java @@ -25,6 +25,8 @@ import java.util.List; import java.util.Set; import java.util.HashSet; +import org.apache.lucene.index.IOContext; + /** * Expert: A Directory instance that switches files between * two other Directory instances. @@ -125,8 +127,8 @@ public class FileSwitchDirectory extends Directory { } @Override - public IndexOutput createOutput(String name) throws IOException { - return getDirectory(name).createOutput(name); + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return getDirectory(name).createOutput(name, context); } @Override @@ -145,7 +147,7 @@ public class FileSwitchDirectory extends Directory { } @Override - public IndexInput openInput(String name) throws IOException { - return getDirectory(name).openInput(name); + public IndexInput openInput(String name, IOContext context) throws IOException { + return getDirectory(name).openInput(name, context); } } diff --git a/lucene/src/java/org/apache/lucene/store/MMapDirectory.java b/lucene/src/java/org/apache/lucene/store/MMapDirectory.java index 19545e951c0..3c18d44e3a5 100644 --- a/lucene/src/java/org/apache/lucene/store/MMapDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/MMapDirectory.java @@ -31,6 +31,7 @@ import java.security.PrivilegedExceptionAction; import java.security.PrivilegedActionException; import java.lang.reflect.Method; +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.Constants; /** File-based {@link Directory} implementation that uses @@ -208,7 +209,7 @@ public class MMapDirectory extends FSDirectory { /** Creates an IndexInput for the file with the given name. */ @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); File f = new File(getDirectory(), name); RandomAccessFile raf = new RandomAccessFile(f, "r"); @@ -399,4 +400,5 @@ public class MMapDirectory extends FSDirectory { } } } + } diff --git a/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java b/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java index b70c9ceebc2..d128f27e4e3 100644 --- a/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/NIOFSDirectory.java @@ -23,6 +23,7 @@ import java.nio.ByteBuffer; import java.nio.channels.ClosedChannelException; // javadoc @link import java.nio.channels.FileChannel; import java.util.concurrent.Future; // javadoc +import org.apache.lucene.index.IOContext; /** * An {@link FSDirectory} implementation that uses java.nio's FileChannel's @@ -73,9 +74,9 @@ public class NIOFSDirectory extends FSDirectory { /** Creates an IndexInput for the file with the given name. */ @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new NIOFSIndexInput(new File(getDirectory(), name), bufferSize, getReadChunkSize()); + return new NIOFSIndexInput(new File(getDirectory(), name), context, getReadChunkSize()); } protected static class NIOFSIndexInput extends SimpleFSDirectory.SimpleFSIndexInput { @@ -87,8 +88,8 @@ public class NIOFSDirectory extends FSDirectory { final FileChannel channel; - public NIOFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException { - super(path, bufferSize, chunkSize); + public NIOFSIndexInput(File path, IOContext context, int chunkSize) throws IOException { + super(path, context, chunkSize); channel = file.getChannel(); } @@ -178,4 +179,5 @@ public class NIOFSDirectory extends FSDirectory { } } } + } diff --git a/lucene/src/java/org/apache/lucene/store/RAMDirectory.java b/lucene/src/java/org/apache/lucene/store/RAMDirectory.java index e33af1a5ee6..ae6b85a891b 100644 --- a/lucene/src/java/org/apache/lucene/store/RAMDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/RAMDirectory.java @@ -27,6 +27,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicLong; +import org.apache.lucene.index.IOContext; + /** * A memory-resident {@link Directory} implementation. Locking * implementation is by default the {@link SingleInstanceLockFactory} @@ -38,7 +40,7 @@ public class RAMDirectory extends Directory { // ***** // Lock acquisition sequence: RAMDirectory, then RAMFile - // ***** + // ***** /** Constructs an empty {@link Directory}. */ public RAMDirectory() { @@ -65,14 +67,14 @@ public class RAMDirectory extends Directory { * @param dir a Directory value * @exception IOException if an error occurs */ - public RAMDirectory(Directory dir) throws IOException { - this(dir, false); + public RAMDirectory(Directory dir, IOContext context) throws IOException { + this(dir, false, context); } - private RAMDirectory(Directory dir, boolean closeDir) throws IOException { + private RAMDirectory(Directory dir, boolean closeDir, IOContext context) throws IOException { this(); for (String file : dir.listAll()) { - dir.copy(this, file, file); + dir.copy(this, file, file, context); } if (closeDir) { dir.close(); @@ -149,7 +151,7 @@ public class RAMDirectory extends Directory { /** Creates a new, empty file in the directory with the given name. Returns a stream writing this file. */ @Override - public IndexOutput createOutput(String name) throws IOException { + public IndexOutput createOutput(String name, IOContext context) throws IOException { ensureOpen(); RAMFile file = newRAMFile(); RAMFile existing = fileMap.remove(name); @@ -176,7 +178,7 @@ public class RAMDirectory extends Directory { /** Returns a stream reading an existing file. */ @Override - public IndexInput openInput(String name) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); RAMFile file = fileMap.get(name); if (file == null) { diff --git a/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java b/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java index 164d0c17bed..d466ec38d0f 100644 --- a/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java +++ b/lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java @@ -21,6 +21,8 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; +import org.apache.lucene.index.IOContext; + /** A straightforward implementation of {@link FSDirectory} * using java.io.RandomAccessFile. However, this class has * poor concurrent performance (multiple threads will @@ -51,9 +53,9 @@ public class SimpleFSDirectory extends FSDirectory { /** Creates an IndexInput for the file with the given name. */ @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { ensureOpen(); - return new SimpleFSIndexInput(new File(directory, name), bufferSize, getReadChunkSize()); + return new SimpleFSIndexInput(new File(directory, name), context, getReadChunkSize()); } protected static class SimpleFSIndexInput extends BufferedIndexInput { @@ -85,8 +87,9 @@ public class SimpleFSDirectory extends FSDirectory { // LUCENE-1566 - maximum read length on a 32bit JVM to prevent incorrect OOM protected final int chunkSize; - public SimpleFSIndexInput(File path, int bufferSize, int chunkSize) throws IOException { - super(bufferSize); + public SimpleFSIndexInput(File path, IOContext context, int chunkSize) throws IOException { + //nocommit Use IOContext to decide bufferSize instead of BufferedIndexInput.BUFFER_SIZE + super(BufferedIndexInput.BUFFER_SIZE); file = new Descriptor(path, "r"); this.chunkSize = chunkSize; } diff --git a/lucene/src/java/org/apache/lucene/util/BitVector.java b/lucene/src/java/org/apache/lucene/util/BitVector.java index f9dbeb36542..f182c9fc4c2 100644 --- a/lucene/src/java/org/apache/lucene/util/BitVector.java +++ b/lucene/src/java/org/apache/lucene/util/BitVector.java @@ -19,6 +19,9 @@ package org.apache.lucene.util; import java.io.IOException; +import org.apache.lucene.index.IOContext; +import org.apache.lucene.index.IOContext.Context; +import org.apache.lucene.index.MergePolicy.OneMerge; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexInput; import org.apache.lucene.store.IndexOutput; @@ -162,8 +165,8 @@ public final class BitVector implements Cloneable, Bits { /** Writes this vector to the file name in Directory d, in a format that can be read by the constructor {@link #BitVector(Directory, String)}. */ - public final void write(Directory d, String name) throws IOException { - IndexOutput output = d.createOutput(name); + public final void write(Directory d, String name, IOContext context) throws IOException { + IndexOutput output = d.createOutput(name, context); try { if (isSparse()) { writeDgaps(output); // sparse bit-set more efficiently saved as d-gaps. @@ -220,8 +223,8 @@ public final class BitVector implements Cloneable, Bits { /** Constructs a bit vector from the file name in Directory d, as written by the {@link #write} method. */ - public BitVector(Directory d, String name) throws IOException { - IndexInput input = d.openInput(name); + public BitVector(Directory d, String name, IOContext context) throws IOException { + IndexInput input = d.openInput(name, context); try { size = input.readInt(); // read size if (size == -1) { diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java index 614a972af2a..c8c9db24e42 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.mockintblock; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; @@ -84,8 +85,8 @@ public class MockFixedIntBlockCodec extends Codec { } @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { - return new FixedIntBlockIndexInput(dir.openInput(fileName, readBufferSize)) { + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return new FixedIntBlockIndexInput(dir.openInput(fileName, context)) { @Override protected BlockReader getBlockReader(final IndexInput in, final int[] buffer) throws IOException { @@ -103,7 +104,7 @@ public class MockFixedIntBlockCodec extends Codec { @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { - IndexOutput out = dir.createOutput(fileName); + IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { FixedIntBlockIndexOutput ret = new FixedIntBlockIndexOutput(out, blockSize) { @@ -160,7 +161,7 @@ public class MockFixedIntBlockCodec extends Codec { public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - state.readBufferSize, + state.context, new MockIntFactory(blockSize), state.codecId); TermsIndexReaderBase indexReader; @@ -170,7 +171,8 @@ public class MockFixedIntBlockCodec extends Codec { state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId); + BytesRef.getUTF8SortedAsUnicodeComparator(), state.codecId, + IOContext.DEFAULT); success = true; } finally { if (!success) { @@ -185,7 +187,7 @@ public class MockFixedIntBlockCodec extends Codec { state.fieldInfos, state.segmentInfo.name, postingsReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java index f3567f76dd5..f4ba2f2b341 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java @@ -20,6 +20,7 @@ package org.apache.lucene.index.codecs.mockintblock; import java.io.IOException; import java.util.Set; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; import org.apache.lucene.index.SegmentWriteState; @@ -82,8 +83,8 @@ public class MockVariableIntBlockCodec extends Codec { } @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { - final IndexInput in = dir.openInput(fileName, readBufferSize); + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + final IndexInput in = dir.openInput(fileName, context); final int baseBlockSize = in.readInt(); return new VariableIntBlockIndexInput(in) { @@ -107,7 +108,7 @@ public class MockVariableIntBlockCodec extends Codec { @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { - final IndexOutput out = dir.createOutput(fileName); + final IndexOutput out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { out.writeInt(baseBlockSize); @@ -182,7 +183,7 @@ public class MockVariableIntBlockCodec extends Codec { public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - state.readBufferSize, + state.context, new MockIntFactory(baseBlockSize), state.codecId); TermsIndexReaderBase indexReader; @@ -193,7 +194,7 @@ public class MockVariableIntBlockCodec extends Codec { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -208,7 +209,7 @@ public class MockVariableIntBlockCodec extends Codec { state.fieldInfos, state.segmentInfo.name, postingsReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java index 6ad2ad2c896..76a06827a57 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java @@ -25,6 +25,7 @@ import java.util.Random; import java.util.Set; import org.apache.lucene.index.FieldInfo; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.PerDocWriteState; import org.apache.lucene.index.SegmentInfo; @@ -103,14 +104,14 @@ public class MockRandomCodec extends Codec { } @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { // Must only use extension, because IW.addIndexes can // rename segment! final IntStreamFactory f = delegates.get((Math.abs(salt ^ getExtension(fileName).hashCode())) % delegates.size()); if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: read using int factory " + f + " from fileName=" + fileName); } - return f.openInput(dir, fileName, readBufferSize); + return f.openInput(dir, fileName, context); } @Override @@ -140,7 +141,7 @@ public class MockRandomCodec extends Codec { } final String seedFileName = IndexFileNames.segmentFileName(state.segmentName, state.codecId, SEED_EXT); - final IndexOutput out = state.directory.createOutput(seedFileName); + final IndexOutput out = state.directory.createOutput(seedFileName, state.context); try { out.writeLong(seed); } finally { @@ -241,7 +242,7 @@ public class MockRandomCodec extends Codec { public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { final String seedFileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.codecId, SEED_EXT); - final IndexInput in = state.dir.openInput(seedFileName); + final IndexInput in = state.dir.openInput(seedFileName, state.context); final long seed = in.readLong(); if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: reading from seg=" + state.segmentInfo.name + " seed=" + seed); @@ -259,12 +260,12 @@ public class MockRandomCodec extends Codec { if (random.nextBoolean()) { postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - readBufferSize, new MockIntStreamFactory(random), state.codecId); + state.context, new MockIntStreamFactory(random), state.codecId); } else { if (LuceneTestCase.VERBOSE) { System.out.println("MockRandomCodec: reading Standard postings"); } - postingsReader = new StandardPostingsReader(state.dir, state.segmentInfo, readBufferSize, state.codecId); + postingsReader = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); } if (random.nextBoolean()) { @@ -293,7 +294,7 @@ public class MockRandomCodec extends Codec { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId); + state.codecId, state.context); } else { final int n2 = random.nextInt(3); if (n2 == 1) { @@ -311,7 +312,7 @@ public class MockRandomCodec extends Codec { state.fieldInfos, state.segmentInfo.name, state.termsIndexDivisor, - state.codecId); + state.codecId, state.context); } success = true; } finally { @@ -329,7 +330,7 @@ public class MockRandomCodec extends Codec { state.fieldInfos, state.segmentInfo.name, postingsReader, - readBufferSize, + state.context, termsCacheSize, state.codecId); success = true; diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java index f1c99b620a0..d2f4b413a2c 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java @@ -93,7 +93,7 @@ public class MockSepCodec extends Codec { public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { PostingsReaderBase postingsReader = new SepPostingsReaderImpl(state.dir, state.segmentInfo, - state.readBufferSize, new MockSingleIntFactory(), state.codecId); + state.context, new MockSingleIntFactory(), state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -103,7 +103,7 @@ public class MockSepCodec extends Codec { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -118,7 +118,7 @@ public class MockSepCodec extends Codec { state.fieldInfos, state.segmentInfo.name, postingsReader, - state.readBufferSize, + state.context, StandardCodec.TERMS_CACHE_SIZE, state.codecId); success = true; diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java index 092db1269cd..debc21741ff 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntFactory.java @@ -18,6 +18,7 @@ package org.apache.lucene.index.codecs.mocksep; */ import org.apache.lucene.store.Directory; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.sep.IntStreamFactory; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.index.codecs.sep.IntIndexOutput; @@ -27,8 +28,8 @@ import java.io.IOException; /** @lucene.experimental */ public class MockSingleIntFactory extends IntStreamFactory { @Override - public IntIndexInput openInput(Directory dir, String fileName, int readBufferSize) throws IOException { - return new MockSingleIntIndexInput(dir, fileName, readBufferSize); + public IntIndexInput openInput(Directory dir, String fileName, IOContext context) throws IOException { + return new MockSingleIntIndexInput(dir, fileName, context); } @Override public IntIndexOutput createOutput(Directory dir, String fileName) throws IOException { diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java index d5d45bf1c2a..933da1861cc 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexInput.java @@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs.mocksep; import java.io.IOException; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.sep.IntIndexInput; import org.apache.lucene.store.DataInput; import org.apache.lucene.store.Directory; @@ -35,9 +36,9 @@ import org.apache.lucene.util.CodecUtil; public class MockSingleIntIndexInput extends IntIndexInput { private final IndexInput in; - public MockSingleIntIndexInput(Directory dir, String fileName, int readBufferSize) + public MockSingleIntIndexInput(Directory dir, String fileName, IOContext context) throws IOException { - in = dir.openInput(fileName, readBufferSize); + in = dir.openInput(fileName, context); CodecUtil.checkHeader(in, MockSingleIntIndexOutput.CODEC, MockSingleIntIndexOutput.VERSION_START, MockSingleIntIndexOutput.VERSION_START); diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java index 3deb1d4b852..a40fa5fde6c 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSingleIntIndexOutput.java @@ -21,6 +21,7 @@ import org.apache.lucene.store.IndexOutput; import org.apache.lucene.store.Directory; import org.apache.lucene.util.CodecUtil; import org.apache.lucene.util.IOUtils; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.sep.IntIndexOutput; import java.io.IOException; @@ -36,7 +37,8 @@ public class MockSingleIntIndexOutput extends IntIndexOutput { final static int VERSION_CURRENT = VERSION_START; public MockSingleIntIndexOutput(Directory dir, String fileName) throws IOException { - out = dir.createOutput(fileName); + //nocommit pass IOContext in via ctor! + out = dir.createOutput(fileName, IOContext.DEFAULT); boolean success = false; try { CodecUtil.writeHeader(out, CODEC, VERSION_CURRENT); diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java index 390d10d9f0d..4950cf97ea9 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java @@ -50,12 +50,12 @@ class PreFlexFieldsWriter extends FieldsConsumer { state.termIndexInterval); final String freqFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.FREQ_EXTENSION); - freqOut = state.directory.createOutput(freqFile); + freqOut = state.directory.createOutput(freqFile, state.context); totalNumDocs = state.numDocs; if (state.fieldInfos.hasProx()) { final String proxFile = IndexFileNames.segmentFileName(state.segmentName, "", PreFlexCodec.PROX_EXTENSION); - proxOut = state.directory.createOutput(proxFile); + proxOut = state.directory.createOutput(proxFile, state.context); } else { proxOut = null; } diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java index 5a2d9477216..d2edcca41b7 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/PreFlexRWCodec.java @@ -51,7 +51,7 @@ public class PreFlexRWCodec extends PreFlexCodec { // Whenever IW opens readers, eg for merging, we have to // keep terms order in UTF16: - return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.readBufferSize, state.termsIndexDivisor) { + return new PreFlexFields(state.dir, state.fieldInfos, state.segmentInfo, state.context, state.termsIndexDivisor) { @Override protected boolean sortTermsByUnicode() { // We carefully peek into stack track above us: if diff --git a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java index d3e00255efb..de60280bed9 100644 --- a/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java +++ b/lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java @@ -22,6 +22,7 @@ import java.io.Closeable; import java.io.IOException; import org.apache.lucene.index.FieldInfos; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexFileNames; import org.apache.lucene.index.codecs.preflex.PreFlexCodec; import org.apache.lucene.index.codecs.preflex.TermInfo; @@ -123,9 +124,10 @@ final class TermInfosWriter implements Closeable { indexInterval = interval; fieldInfos = fis; isIndex = isi; + // nocommit pass IOContext in via ctor output = directory.createOutput(IndexFileNames.segmentFileName(segment, "", (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION - : PreFlexCodec.TERMS_EXTENSION))); + : PreFlexCodec.TERMS_EXTENSION)), IOContext.DEFAULT); boolean success = false; try { output.writeInt(FORMAT_CURRENT); // write format diff --git a/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java b/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java index 922cef87999..d4a503355d1 100644 --- a/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java +++ b/lucene/src/test-framework/org/apache/lucene/store/MockDirectoryWrapper.java @@ -31,6 +31,7 @@ import java.util.Map; import java.util.Random; import java.util.Set; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.codecs.CodecProvider; import org.apache.lucene.util.LuceneTestCase; @@ -196,7 +197,8 @@ public class MockDirectoryWrapper extends Directory { long length = fileLength(name); byte[] zeroes = new byte[256]; long upto = 0; - IndexOutput out = delegate.createOutput(name); + //nocommit - randomize the IOContext here? + IndexOutput out = delegate.createOutput(name, IOContext.DEFAULT); while(upto < length) { final int limit = (int) Math.min(length-upto, zeroes.length); out.writeBytes(zeroes, 0, limit); @@ -205,7 +207,8 @@ public class MockDirectoryWrapper extends Directory { out.close(); } else if (count % 3 == 2) { // Truncate the file: - IndexOutput out = delegate.createOutput(name); + //nocommit - randomize the IOContext here? + IndexOutput out = delegate.createOutput(name, IOContext.DEFAULT); out.setLength(fileLength(name)/2); out.close(); } @@ -337,7 +340,7 @@ public class MockDirectoryWrapper extends Directory { } @Override - public synchronized IndexOutput createOutput(String name) throws IOException { + public synchronized IndexOutput createOutput(String name, IOContext context) throws IOException { maybeYield(); if (crashed) throw new IOException("cannot createOutput after crash"); @@ -372,7 +375,8 @@ public class MockDirectoryWrapper extends Directory { } //System.out.println(Thread.currentThread().getName() + ": MDW: create " + name); - IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name), name); + // nocommit - randomize the IOContext here? + IndexOutput io = new MockIndexOutputWrapper(this, delegate.createOutput(name, context), name); addFileHandle(io, name, false); openFilesForWrite.add(name); @@ -401,7 +405,7 @@ public class MockDirectoryWrapper extends Directory { } @Override - public synchronized IndexInput openInput(String name) throws IOException { + public synchronized IndexInput openInput(String name, IOContext context) throws IOException { maybeYield(); if (!delegate.fileExists(name)) throw new FileNotFoundException(name); @@ -412,7 +416,8 @@ public class MockDirectoryWrapper extends Directory { throw fillOpenTrace(new IOException("MockDirectoryWrapper: file \"" + name + "\" is still open for writing"), name, false); } - IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name)); + // nocommit - randomize IOContext here? + IndexInput ii = new MockIndexInputWrapper(this, name, delegate.openInput(name, context)); addFileHandle(ii, name, true); return ii; } @@ -637,9 +642,10 @@ public class MockDirectoryWrapper extends Directory { } @Override - public synchronized void copy(Directory to, String src, String dest) throws IOException { + public synchronized void copy(Directory to, String src, String dest, IOContext context) throws IOException { maybeYield(); - delegate.copy(to, src, dest); + // randomize the IOContext here? + delegate.copy(to, src, dest, context); } } diff --git a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java index 6dd4059842e..a109f8f7f58 100644 --- a/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java +++ b/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java @@ -1056,7 +1056,8 @@ public abstract class LuceneTestCase extends Assert { public static MockDirectoryWrapper newDirectory(Random r, Directory d) throws IOException { Directory impl = newDirectoryImpl(r, TEST_DIRECTORY); for (String file : d.listAll()) { - d.copy(impl, file, file); + //nocommit randomiz the IOContext here? + d.copy(impl, file, file, IOContext.DEFAULT); } MockDirectoryWrapper dir = new MockDirectoryWrapper(r, impl); stores.put(dir, Thread.currentThread().getStackTrace()); diff --git a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java index 7bc484c931e..aa9d2e88706 100755 --- a/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java +++ b/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java @@ -395,7 +395,7 @@ public class TestAddIndexes extends LuceneTestCase { setMergePolicy(newLogMergePolicy(4)) ); - writer.addIndexes(aux, new MockDirectoryWrapper(random, new RAMDirectory(aux))); + writer.addIndexes(aux, new MockDirectoryWrapper(random, new RAMDirectory(aux, IOContext.DEFAULT))); assertEquals(1060, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -430,7 +430,7 @@ public class TestAddIndexes extends LuceneTestCase { setMergePolicy(newLogMergePolicy(4)) ); - writer.addIndexes(aux, new MockDirectoryWrapper(random, new RAMDirectory(aux))); + writer.addIndexes(aux, new MockDirectoryWrapper(random, new RAMDirectory(aux, IOContext.DEFAULT))); assertEquals(1020, writer.maxDoc()); assertEquals(1000, writer.getDocCount(0)); writer.close(); @@ -665,7 +665,7 @@ public class TestAddIndexes extends LuceneTestCase { final Directory[] dirs = new Directory[NUM_COPY]; for(int k=0;k filesToDelete = merger.createCompoundFile(merged + ".cfs", info); + Collection filesToDelete = merger.createCompoundFile(merged + ".cfs", info, IOContext.DEFAULT); info.setUseCompoundFile(true); for (final String fileToDelete : filesToDelete) si1.dir.deleteFile(fileToDelete); @@ -220,7 +223,7 @@ public class TestDoc extends LuceneTestCase { private void printSegment(PrintWriter out, SegmentInfo si) throws Exception { - SegmentReader reader = SegmentReader.get(true, si, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + SegmentReader reader = SegmentReader.get(true, si, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); for (int i = 0; i < reader.numDocs(); i++) out.println(reader.document(i)); diff --git a/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java b/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java index cce37775907..93ba1510731 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java +++ b/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java @@ -148,7 +148,7 @@ public class TestDocTermOrds extends LuceneTestCase { @Override public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException { - PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.readBufferSize, state.codecId); + PostingsReaderBase postings = new StandardPostingsReader(state.dir, state.segmentInfo, state.context, state.codecId); TermsIndexReaderBase indexReader; boolean success = false; @@ -158,7 +158,7 @@ public class TestDocTermOrds extends LuceneTestCase { state.segmentInfo.name, state.termsIndexDivisor, BytesRef.getUTF8SortedAsUnicodeComparator(), - state.codecId); + state.codecId, state.context); success = true; } finally { if (!success) { @@ -173,7 +173,7 @@ public class TestDocTermOrds extends LuceneTestCase { state.fieldInfos, state.segmentInfo.name, postings, - state.readBufferSize, + state.context, TERMS_CACHE_SIZE, state.codecId); success = true; diff --git a/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java b/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java index f3335b55d2d..2c4651ba636 100644 --- a/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestDocumentWriter.java @@ -34,6 +34,7 @@ import org.apache.lucene.document.Field.Index; import org.apache.lucene.document.Field.Store; import org.apache.lucene.document.Field.TermVector; import org.apache.lucene.document.Fieldable; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.Directory; import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; @@ -68,7 +69,7 @@ public class TestDocumentWriter extends LuceneTestCase { SegmentInfo info = writer.newestSegment(); writer.close(); //After adding the document, we should be able to read it back in - SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); assertTrue(reader != null); Document doc = reader.document(0); assertTrue(doc != null); @@ -129,7 +130,7 @@ public class TestDocumentWriter extends LuceneTestCase { writer.commit(); SegmentInfo info = writer.newestSegment(); writer.close(); - SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); DocsAndPositionsEnum termPositions = MultiFields.getTermPositionsEnum(reader, MultiFields.getDeletedDocs(reader), "repeated", new BytesRef("repeated")); @@ -193,7 +194,7 @@ public class TestDocumentWriter extends LuceneTestCase { writer.commit(); SegmentInfo info = writer.newestSegment(); writer.close(); - SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); DocsAndPositionsEnum termPositions = reader.fields().terms("f1").docsAndPositions(reader.getDeletedDocs(), new BytesRef("a"), null); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); @@ -237,7 +238,7 @@ public class TestDocumentWriter extends LuceneTestCase { writer.commit(); SegmentInfo info = writer.newestSegment(); writer.close(); - SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR); + SegmentReader reader = SegmentReader.get(true, info, IndexReader.DEFAULT_TERMS_INDEX_DIVISOR, IOContext.READ); DocsAndPositionsEnum termPositions = reader.fields().terms("preanalyzed").docsAndPositions(reader.getDeletedDocs(), new BytesRef("term1"), null); assertTrue(termPositions.nextDoc() != termPositions.NO_MORE_DOCS); diff --git a/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java b/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java index d68afdd3202..ea9cc600835 100644 --- a/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java +++ b/lucene/src/test/org/apache/lucene/index/TestFieldInfos.java @@ -20,6 +20,7 @@ package org.apache.lucene.index; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; import org.apache.lucene.document.Document; +import org.apache.lucene.index.IOContext.Context; import org.apache.lucene.store.Directory; import org.apache.lucene.store.IndexOutput; @@ -47,7 +48,7 @@ public class TestFieldInfos extends LuceneTestCase { assertTrue(fieldInfos.size() == DocHelper.all.size()); //this is all b/c we are using the no-arg constructor - IndexOutput output = dir.createOutput(filename); + IndexOutput output = dir.createOutput(filename, IOContext.DEFAULT); assertTrue(output != null); //Use a RAMOutputStream diff --git a/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java b/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java index 965cbe29534..a1874a150ca 100644 --- a/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java +++ b/lucene/src/test/org/apache/lucene/index/TestFieldsReader.java @@ -398,8 +398,8 @@ public class TestFieldsReader extends LuceneTestCase { lockFactory = fsDir.getLockFactory(); } @Override - public IndexInput openInput(String name) throws IOException { - return new FaultyIndexInput(fsDir.openInput(name)); + public IndexInput openInput(String name, IOContext context) throws IOException { + return new FaultyIndexInput(fsDir.openInput(name, context)); } @Override public String[] listAll() throws IOException { @@ -422,8 +422,8 @@ public class TestFieldsReader extends LuceneTestCase { return fsDir.fileLength(name); } @Override - public IndexOutput createOutput(String name) throws IOException { - return fsDir.createOutput(name); + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return fsDir.createOutput(name, context); } @Override public void sync(Collection names) throws IOException { diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java b/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java index f14abca8bd1..7430423416c 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexFileDeleter.java @@ -91,7 +91,7 @@ public class TestIndexFileDeleter extends LuceneTestCase { // figure out which field number corresponds to // "content", and then set our expected file names below // accordingly: - CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs"); + CompoundFileReader cfsReader = new CompoundFileReader(dir, "_2.cfs", IOContext.DEFAULT); FieldInfos fieldInfos = new FieldInfos(cfsReader, "_2.fnm"); int contentFieldIndex = -1; for (FieldInfo fi : fieldInfos) { @@ -212,8 +212,8 @@ public class TestIndexFileDeleter extends LuceneTestCase { } public void copyFile(Directory dir, String src, String dest) throws IOException { - IndexInput in = dir.openInput(src); - IndexOutput out = dir.createOutput(dest); + IndexInput in = dir.openInput(src, IOContext.DEFAULT); + IndexOutput out = dir.createOutput(dest, IOContext.DEFAULT); byte[] b = new byte[1024]; long remainder = in.length(); while(remainder > 0) { diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexInput.java b/lucene/src/test/org/apache/lucene/index/TestIndexInput.java index c5fc426c810..e593760ce76 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexInput.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexInput.java @@ -99,10 +99,10 @@ public class TestIndexInput extends LuceneTestCase { // this test checks the raw IndexInput methods as it uses RAMIndexInput which extends IndexInput directly public void testRawIndexInputRead() throws IOException { final RAMDirectory dir = new RAMDirectory(); - final IndexOutput os = dir.createOutput("foo"); + final IndexOutput os = dir.createOutput("foo", IOContext.DEFAULT); os.writeBytes(READ_TEST_BYTES, READ_TEST_BYTES.length); os.close(); - final IndexInput is = dir.openInput("foo"); + final IndexInput is = dir.openInput("foo", IOContext.DEFAULT); checkReads(is); is.close(); dir.close(); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java b/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java index d17457fe00a..dfcad0301cb 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexReaderOnDiskFull.java @@ -84,7 +84,7 @@ public class TestIndexReaderOnDiskFull extends LuceneTestCase { // Iterate w/ ever increasing free disk space: while(!done) { - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir)); + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir, IOContext.DEFAULT)); // If IndexReader hits disk full, it can write to // the same files again. diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java index a52ab279d7a..050da2b5ae0 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java @@ -1029,7 +1029,7 @@ public class TestIndexWriter extends LuceneTestCase { Directory dir = newDirectory(); try { // Create my own random file: - IndexOutput out = dir.createOutput("myrandomfile"); + IndexOutput out = dir.createOutput("myrandomfile", IOContext.DEFAULT); out.writeByte((byte) 42); out.close(); diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java index 03b72678579..4cb2d5fefbd 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterDelete.java @@ -456,7 +456,7 @@ public class TestIndexWriterDelete extends LuceneTestCase { if (VERBOSE) { System.out.println("TEST: cycle"); } - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir)); + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir, IOContext.DEFAULT)); dir.setPreventDoubleWrite(false); IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig( diff --git a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java index 83dcc257a0f..67e25c34804 100644 --- a/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java +++ b/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java @@ -935,7 +935,7 @@ public class TestIndexWriterExceptions extends LuceneTestCase { if (VERBOSE) { System.out.println("TEST: iter " + i); } - MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir)); + MockDirectoryWrapper dir = new MockDirectoryWrapper(random, new RAMDirectory(startDir, IOContext.DEFAULT)); conf = newIndexWriterConfig( TEST_VERSION_CURRENT, new MockAnalyzer(random)).setMergeScheduler(new ConcurrentMergeScheduler()); ((ConcurrentMergeScheduler) conf.getMergeScheduler()).setSuppressExceptions(); w = new IndexWriter(dir, conf); @@ -1039,8 +1039,8 @@ public class TestIndexWriterExceptions extends LuceneTestCase { assertTrue("segment generation should be > 0 but got " + gen, gen > 0); final String segmentsFileName = SegmentInfos.getCurrentSegmentFileName(dir); - IndexInput in = dir.openInput(segmentsFileName); - IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen)); + IndexInput in = dir.openInput(segmentsFileName, IOContext.DEFAULT); + IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen), IOContext.DEFAULT); out.copyBytes(in, in.length()-1); byte b = in.readByte(); out.writeByte((byte) (1+b)); @@ -1084,8 +1084,8 @@ public class TestIndexWriterExceptions extends LuceneTestCase { String fileNameOut = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1+gen); - IndexInput in = dir.openInput(fileNameIn); - IndexOutput out = dir.createOutput(fileNameOut); + IndexInput in = dir.openInput(fileNameIn, IOContext.DEFAULT); + IndexOutput out = dir.createOutput(fileNameOut, IOContext.DEFAULT); long length = in.length(); for(int i=0;i set = mapper.getTermVectorEntrySet(); @@ -384,7 +385,7 @@ public class TestTermVectorsReader extends LuceneTestCase { public void testBadParams() throws IOException { TermVectorsReader reader = null; try { - reader = new TermVectorsReader(dir, seg, fieldInfos); + reader = new TermVectorsReader(dir, seg, fieldInfos, IOContext.READ); //Bad document number, good field number reader.get(50, testFields[0]); fail(); @@ -394,7 +395,7 @@ public class TestTermVectorsReader extends LuceneTestCase { reader.close(); } try { - reader = new TermVectorsReader(dir, seg, fieldInfos); + reader = new TermVectorsReader(dir, seg, fieldInfos, IOContext.READ); //Bad document number, no field reader.get(50); fail(); @@ -404,7 +405,7 @@ public class TestTermVectorsReader extends LuceneTestCase { reader.close(); } try { - reader = new TermVectorsReader(dir, seg, fieldInfos); + reader = new TermVectorsReader(dir, seg, fieldInfos, IOContext.READ); //good document number, bad field number TermFreqVector vector = reader.get(0, "f50"); assertTrue(vector == null); diff --git a/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java b/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java index 350a05b6c9b..b6675c6ff92 100644 --- a/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java +++ b/lucene/src/test/org/apache/lucene/index/TestTermVectorsWriter.java @@ -299,7 +299,7 @@ public class TestTermVectorsWriter extends LuceneTestCase { .setMergeScheduler(new SerialMergeScheduler()).setMergePolicy( new LogDocMergePolicy())); - Directory[] indexDirs = {new MockDirectoryWrapper(random, new RAMDirectory(dir))}; + Directory[] indexDirs = {new MockDirectoryWrapper(random, new RAMDirectory(dir, IOContext.DEFAULT))}; writer.addIndexes(indexDirs); writer.optimize(); writer.close(); diff --git a/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java b/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java index 71f155f634f..c97e33c13c9 100644 --- a/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java +++ b/lucene/src/test/org/apache/lucene/index/codecs/intblock/TestIntBlockCodec.java @@ -19,6 +19,7 @@ package org.apache.lucene.index.codecs.intblock; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.store.*; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.codecs.sep.*; import org.apache.lucene.index.codecs.mockintblock.*; @@ -35,7 +36,7 @@ public class TestIntBlockCodec extends LuceneTestCase { } out.close(); - IntIndexInput in = f.openInput(dir, "test"); + IntIndexInput in = f.openInput(dir, "test", IOContext.DEFAULT); IntIndexInput.Reader r = in.reader(); for(int i=0;i<11777;i++) { @@ -55,7 +56,7 @@ public class TestIntBlockCodec extends LuceneTestCase { // write no ints out.close(); - IntIndexInput in = f.openInput(dir, "test"); + IntIndexInput in = f.openInput(dir, "test", IOContext.DEFAULT); in.reader(); // read no ints in.close(); diff --git a/lucene/src/test/org/apache/lucene/search/TestBoolean2.java b/lucene/src/test/org/apache/lucene/search/TestBoolean2.java index c0f4bd33369..396e216ab9e 100644 --- a/lucene/src/test/org/apache/lucene/search/TestBoolean2.java +++ b/lucene/src/test/org/apache/lucene/search/TestBoolean2.java @@ -23,6 +23,7 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.index.Term; import org.apache.lucene.index.IndexReader; @@ -64,13 +65,13 @@ public class TestBoolean2 extends LuceneTestCase { searcher = new IndexSearcher(directory, true); // Make big index - dir2 = new MockDirectoryWrapper(random, new RAMDirectory(directory)); + dir2 = new MockDirectoryWrapper(random, new RAMDirectory(directory, IOContext.DEFAULT)); // First multiply small test index: mulFactor = 1; int docCount = 0; do { - final Directory copy = new MockDirectoryWrapper(random, new RAMDirectory(dir2)); + final Directory copy = new MockDirectoryWrapper(random, new RAMDirectory(dir2, IOContext.DEFAULT)); RandomIndexWriter w = new RandomIndexWriter(random, dir2); w.addIndexes(copy); docCount = w.maxDoc(); diff --git a/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java b/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java index 33e48c4f83d..b421fea941d 100755 --- a/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java +++ b/lucene/src/test/org/apache/lucene/store/TestBufferedIndexInput.java @@ -29,6 +29,7 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -93,11 +94,11 @@ public class TestBufferedIndexInput extends LuceneTestCase { // run test with chunk size of 10 bytes runReadBytesAndClose(new SimpleFSIndexInput(tmpInputFile, - inputBufferSize, 10), inputBufferSize, random); + IOContext.DEFAULT, 10), inputBufferSize, random); // run test with chunk size of 10 bytes runReadBytesAndClose(new NIOFSIndexInput(tmpInputFile, - inputBufferSize, 10), inputBufferSize, random); + IOContext.DEFAULT, 10), inputBufferSize, random); } private void runReadBytesAndClose(IndexInput input, int bufferSize, Random r) @@ -303,11 +304,6 @@ public class TestBufferedIndexInput extends LuceneTestCase { dir = new SimpleFSDirectory(path, null); } - @Override - public IndexInput openInput(String name) throws IOException { - return openInput(name, BufferedIndexInput.BUFFER_SIZE); - } - public void tweakBufferSizes() { //int count = 0; for (final IndexInput ip : allIndexInputs) { @@ -320,17 +316,17 @@ public class TestBufferedIndexInput extends LuceneTestCase { } @Override - public IndexInput openInput(String name, int bufferSize) throws IOException { + public IndexInput openInput(String name, IOContext context) throws IOException { // Make random changes to buffer size - bufferSize = 1+Math.abs(rand.nextInt() % 10); - IndexInput f = dir.openInput(name, bufferSize); + //bufferSize = 1+Math.abs(rand.nextInt() % 10); + IndexInput f = dir.openInput(name, context); allIndexInputs.add(f); return f; } @Override - public IndexOutput createOutput(String name) throws IOException { - return dir.createOutput(name); + public IndexOutput createOutput(String name, IOContext context) throws IOException { + return dir.createOutput(name, context); } @Override diff --git a/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java b/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java index 08f41e46846..94630a0e027 100644 --- a/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java +++ b/lucene/src/test/org/apache/lucene/store/TestCopyBytes.java @@ -18,6 +18,7 @@ package org.apache.lucene.store; */ +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -40,7 +41,7 @@ public class TestCopyBytes extends LuceneTestCase { } // make random file - IndexOutput out = dir.createOutput("test"); + IndexOutput out = dir.createOutput("test", IOContext.DEFAULT); byte[] bytes = new byte[_TestUtil.nextInt(random, 1, 77777)]; final int size = _TestUtil.nextInt(random, 1, 1777777); int upto = 0; @@ -60,9 +61,9 @@ public class TestCopyBytes extends LuceneTestCase { assertEquals(size, dir.fileLength("test")); // copy from test -> test2 - final IndexInput in = dir.openInput("test"); + final IndexInput in = dir.openInput("test", IOContext.DEFAULT); - out = dir.createOutput("test2"); + out = dir.createOutput("test2", IOContext.DEFAULT); upto = 0; while(upto < size) { @@ -80,7 +81,7 @@ public class TestCopyBytes extends LuceneTestCase { in.close(); // verify - IndexInput in2 = dir.openInput("test2"); + IndexInput in2 = dir.openInput("test2", IOContext.DEFAULT); upto = 0; while(upto < size) { if (random.nextBoolean()) { diff --git a/lucene/src/test/org/apache/lucene/store/TestDirectory.java b/lucene/src/test/org/apache/lucene/store/TestDirectory.java index 03e6f150574..9eca6667a16 100644 --- a/lucene/src/test/org/apache/lucene/store/TestDirectory.java +++ b/lucene/src/test/org/apache/lucene/store/TestDirectory.java @@ -17,6 +17,7 @@ package org.apache.lucene.store; * limitations under the License. */ +import org.apache.lucene.index.IOContext; import org.apache.lucene.util.LuceneTestCase; import org.apache.lucene.util._TestUtil; @@ -31,7 +32,7 @@ public class TestDirectory extends LuceneTestCase { for (Directory dir : dirs) { dir.close(); try { - dir.createOutput("test"); + dir.createOutput("test", IOContext.DEFAULT); fail("did not hit expected exception"); } catch (AlreadyClosedException ace) { } @@ -56,7 +57,7 @@ public class TestDirectory extends LuceneTestCase { dir.ensureOpen(); String fname = "foo." + i; String lockname = "foo" + i + ".lck"; - IndexOutput out = dir.createOutput(fname); + IndexOutput out = dir.createOutput(fname, IOContext.DEFAULT); out.writeByte((byte)i); out.close(); @@ -70,7 +71,7 @@ public class TestDirectory extends LuceneTestCase { // closed and will cause a failure to delete the file. if (d2 instanceof MMapDirectory) continue; - IndexInput input = d2.openInput(fname); + IndexInput input = d2.openInput(fname, IOContext.DEFAULT); assertEquals((byte)i, input.readByte()); input.close(); } @@ -141,7 +142,7 @@ public class TestDirectory extends LuceneTestCase { private void checkDirectoryFilter(Directory dir) throws IOException { String name = "file"; try { - dir.createOutput(name).close(); + dir.createOutput(name, IOContext.DEFAULT).close(); assertTrue(dir.fileExists(name)); assertTrue(Arrays.asList(dir.listAll()).contains(name)); } finally { @@ -156,7 +157,7 @@ public class TestDirectory extends LuceneTestCase { path.mkdirs(); new File(path, "subdir").mkdirs(); Directory fsDir = new SimpleFSDirectory(path, null); - assertEquals(0, new RAMDirectory(fsDir).listAll().length); + assertEquals(0, new RAMDirectory(fsDir, IOContext.DEFAULT).listAll().length); } finally { _TestUtil.rmDir(path); } @@ -167,7 +168,7 @@ public class TestDirectory extends LuceneTestCase { File path = _TestUtil.getTempDir("testnotdir"); Directory fsDir = new SimpleFSDirectory(path, null); try { - IndexOutput out = fsDir.createOutput("afile"); + IndexOutput out = fsDir.createOutput("afile", IOContext.DEFAULT); out.close(); assertTrue(fsDir.fileExists("afile")); try { diff --git a/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java b/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java index 7669bb10d41..2a6b4ae9b44 100644 --- a/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java +++ b/lucene/src/test/org/apache/lucene/store/TestMultiMMap.java @@ -23,6 +23,7 @@ import java.util.Random; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.RandomIndexWriter; import org.apache.lucene.util.BytesRef; @@ -51,9 +52,9 @@ public class TestMultiMMap extends LuceneTestCase { for (int i = 0; i < 31; i++) { MMapDirectory mmapDir = new MMapDirectory(_TestUtil.getTempDir("testSeekZero")); mmapDir.setMaxChunkSize(1<=count1; i--) { - BitVector bv2 = new BitVector(d, "TESTBV"); + BitVector bv2 = new BitVector(d, "TESTBV", IOContext.DEFAULT); assertTrue(doCompare(bv,bv2)); bv = bv2; bv.clear(i); assertEquals(i,bv.count()); - bv.write(d, "TESTBV"); + bv.write(d, "TESTBV", IOContext.DEFAULT); } } /** diff --git a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java index d1abb262252..ccee2e662cd 100644 --- a/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java +++ b/lucene/src/test/org/apache/lucene/util/fst/TestFSTs.java @@ -29,6 +29,7 @@ import java.util.*; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; +import org.apache.lucene.index.IOContext; import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.IndexWriter; import org.apache.lucene.index.IndexWriterConfig; @@ -468,10 +469,10 @@ public class TestFSTs extends LuceneTestCase { FST fst = builder.finish(); if (random.nextBoolean() && fst != null) { - IndexOutput out = dir.createOutput("fst.bin"); + IndexOutput out = dir.createOutput("fst.bin", IOContext.DEFAULT); fst.save(out); out.close(); - IndexInput in = dir.openInput("fst.bin"); + IndexInput in = dir.openInput("fst.bin", IOContext.DEFAULT); try { fst = new FST(in, outputs); } finally { @@ -1181,7 +1182,7 @@ public class TestFSTs extends LuceneTestCase { } Directory dir = FSDirectory.open(new File(dirOut)); - IndexOutput out = dir.createOutput("fst.bin"); + IndexOutput out = dir.createOutput("fst.bin", IOContext.DEFAULT); fst.save(out); out.close(); @@ -1510,11 +1511,11 @@ public class TestFSTs extends LuceneTestCase { // Make sure it still works after save/load: Directory dir = newDirectory(); - IndexOutput out = dir.createOutput("fst"); + IndexOutput out = dir.createOutput("fst", IOContext.DEFAULT); fst.save(out); out.close(); - IndexInput in = dir.openInput("fst"); + IndexInput in = dir.openInput("fst", IOContext.DEFAULT); final FST fst2 = new FST(in, outputs); checkStopNodes(fst2, outputs); in.close(); diff --git a/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java b/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java index 6fdc6c85668..6072e636247 100644 --- a/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java +++ b/lucene/src/test/org/apache/lucene/util/packed/TestPackedInts.java @@ -17,6 +17,7 @@ package org.apache.lucene.util.packed; * limitations under the License. */ +import org.apache.lucene.index.IOContext; import org.apache.lucene.store.*; import org.apache.lucene.util.LuceneTestCase; @@ -54,7 +55,7 @@ public class TestPackedInts extends LuceneTestCase { final int valueCount = 100+random.nextInt(500); final Directory d = newDirectory(); - IndexOutput out = d.createOutput("out.bin"); + IndexOutput out = d.createOutput("out.bin", IOContext.DEFAULT); PackedInts.Writer w = PackedInts.getWriter( out, valueCount, nbits); @@ -71,7 +72,7 @@ public class TestPackedInts extends LuceneTestCase { final long fp = out.getFilePointer(); out.close(); {// test reader - IndexInput in = d.openInput("out.bin"); + IndexInput in = d.openInput("out.bin", IOContext.DEFAULT); PackedInts.Reader r = PackedInts.getReader(in); assertEquals(fp, in.getFilePointer()); for(int i=0;i