LUCENE-2793: next iteration on IOContext

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/LUCENE2793@1143677 13f79535-47bb-0310-9956-ffa450edef68

lucene/contrib/misc/src/java/org/apache/lucene/store/DirectIOLinuxDirectory.java

@@ -71,16 +71,20 @@ public class DirectIOLinuxDirectory extends FSDirectory {
   @Override
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
-    //nocommit - use buffer based on IOContext
-    return new DirectIOLinuxIndexInput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexInput.BUFFER_SIZE : forcedBufferSize);
+    return new DirectIOLinuxIndexInput(new File(getDirectory(), name),
+        bufferSize(context));
   }
 
   @Override
-  public IndexOutput createOutput(String name,IOContext context) throws IOException {
+  public IndexOutput createOutput(String name, IOContext context) throws IOException {
     ensureOpen();
     ensureCanWrite(name);
-    //nocommit - use buffer based on IOContext
-    return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), forcedBufferSize == 0 ? BufferedIndexOutput.BUFFER_SIZE : forcedBufferSize);
+    return new DirectIOLinuxIndexOutput(new File(getDirectory(), name), bufferSize(context));
+  }
+  
+  private int bufferSize(IOContext context) {
+    return forcedBufferSize != 0 ? forcedBufferSize : BufferedIndexInput
+        .bufferSize(context);
   }
 
   private final static class DirectIOLinuxIndexOutput extends IndexOutput {
@@ -240,6 +244,7 @@ public class DirectIOLinuxDirectory extends FSDirectory {
     private int bufferPos;
 
     public DirectIOLinuxIndexInput(File path, int bufferSize) throws IOException {
+      // TODO make use of IOContext
       FileDescriptor fd = NativePosixUtil.open_direct(path.toString(), true);
       fis = new FileInputStream(fd);
       channel = fis.getChannel();

lucene/contrib/misc/src/java/org/apache/lucene/store/WindowsDirectory.java

@@ -70,7 +70,6 @@ public class WindowsDirectory extends FSDirectory {
   @Override
   public IndexInput openInput(String name, IOContext context) throws IOException {
     ensureOpen();
-    //nocommit - use buffer based on IOContext
     return new WindowsIndexInput(new File(getDirectory(), name), DEFAULT_BUFFERSIZE);
   }
   

lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java

@@ -274,8 +274,7 @@ class BufferedDeletesStream {
         if (coalescedDeletes != null) {
           // Lock order: IW -> BD -> RP
           assert readerPool.infoIsLive(info);
-          //nocommit is IOContext.DEFAULT the right thing to do here?
-          SegmentReader reader = readerPool.get(info, false, IOContext.DEFAULT);
+          SegmentReader reader = readerPool.get(info, false, IOContext.READ);
           int delCount = 0;
           final boolean segAllDeletes;
           try {

lucene/src/java/org/apache/lucene/index/CompoundFileReader.java

@@ -143,8 +143,7 @@ public class CompoundFileReader extends Directory {
     final FileEntry entry = entries.get(id);
     if (entry == null)
       throw new IOException("No sub-file with id " + id + " found (files: " + entries.keySet() + ")");
-    // nocommit set read buffer size based on IOContext
-    return new CSIndexInput(stream, entry.offset, entry.length, BufferedIndexInput.BUFFER_SIZE);
+    return new CSIndexInput(stream, entry.offset, entry.length, context);
   }
   
   /** Returns an array of strings, one for each file in the directory. */
@@ -221,13 +220,9 @@ public class CompoundFileReader extends Directory {
     IndexInput base;
     long fileOffset;
     long length;
-    
-    CSIndexInput(final IndexInput base, final long fileOffset, final long length) {
-      this(base, fileOffset, length, BufferedIndexInput.BUFFER_SIZE);
-    }
-    
-    CSIndexInput(final IndexInput base, final long fileOffset, final long length, int readBufferSize) {
-      super(readBufferSize);
+
+    CSIndexInput(final IndexInput base, final long fileOffset, final long length, IOContext context) {
+      super(context);
       this.base = (IndexInput)base.clone();
       this.fileOffset = fileOffset;
       this.length = length;

lucene/src/java/org/apache/lucene/index/IndexWriter.java

@@ -634,7 +634,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
       //        readBufferSize = BufferedIndexInput.BUFFER_SIZE;
       //      }
       
-      // nocommit context should be part of the key used to cache that reader in the pool.
+      // TODO: context should be part of the key used to cache that reader in the pool.
 
       SegmentReader sr = readerMap.get(info);
       if (sr == null) {
@@ -3580,8 +3580,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
       // keep deletes (it's costly to open entire reader
       // when we just need deletes)
 
-      // nocommit  should we use another flag "isMergedSegment" or a "READ" context here?
-      
       final SegmentReader mergedReader = readerPool.get(merge.info, loadDocStores, context, termsIndexDivisor);
       try {
         if (poolReaders && mergedSegmentWarmer != null) {

lucene/src/java/org/apache/lucene/index/SegmentNorms.java

@@ -220,7 +220,6 @@ final class SegmentNorms implements Cloneable {
     // NOTE: norms are re-written in regular directory, not cfs
     si.advanceNormGen(this.number);
     final String normFileName = si.getNormFileName(this.number);
-    //nocommit not sure if this is the correct information provided to FlushInfo
     IndexOutput out = owner.directory().createOutput(normFileName, new IOContext(new FlushInfo(si.docCount, 0)));
     boolean success = false;
     try {

lucene/src/java/org/apache/lucene/index/codecs/DefaultSegmentInfosWriter.java

@@ -23,6 +23,7 @@ import org.apache.lucene.index.SegmentInfo;
 import org.apache.lucene.index.SegmentInfos;
 import org.apache.lucene.store.ChecksumIndexOutput;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.store.FlushInfo;
 import org.apache.lucene.store.IOContext;
 import org.apache.lucene.store.IndexOutput;
 import org.apache.lucene.util.IOUtils;
@@ -57,8 +58,7 @@ public class DefaultSegmentInfosWriter extends SegmentInfosWriter {
   @Override
   public IndexOutput writeInfos(Directory dir, String segmentFileName, SegmentInfos infos, IOContext context)
           throws IOException {
-    //nocommit should this context always be flush?
-    IndexOutput out = createOutput(dir, segmentFileName, context);
+    IndexOutput out = createOutput(dir, segmentFileName, new IOContext(new FlushInfo(infos.size(), infos.totalDocCount())));
     boolean success = false;
     try {
       out.writeInt(FORMAT_CURRENT); // write FORMAT

lucene/src/java/org/apache/lucene/store/BufferedIndexInput.java

@@ -22,8 +22,19 @@ import java.io.IOException;
 /** Base implementation class for buffered {@link IndexInput}. */
 public abstract class BufferedIndexInput extends IndexInput {
 
-  /** Default buffer size */
+  /** Default buffer size set to 1024*/
   public static final int BUFFER_SIZE = 1024;
+  
+  // The normal read buffer size defaults to 1024, but
+  // increasing this during merging seems to yield
+  // performance gains.  However we don't want to increase
+  // it too much because there are quite a few
+  // BufferedIndexInputs created during merging.  See
+  // LUCENE-888 for details.
+  /**
+   * A buffer size for merges set to 4096
+   */
+  public static final int MERGE_BUFFER_SIZE = 4096;
 
   private int bufferSize = BUFFER_SIZE;
   
@@ -41,11 +52,14 @@ public abstract class BufferedIndexInput extends IndexInput {
   }
 
   public BufferedIndexInput() {}
+  
+  public BufferedIndexInput(IOContext context) {
+    this(bufferSize(context));
+  }
 
   /** Inits BufferedIndexInput with a specific bufferSize */
-  //nocommit To cleanup class variable bufferSize as the the default size is always used 
   public BufferedIndexInput(int bufferSize) {
-    checkBufferSize(BufferedIndexInput.BUFFER_SIZE);
+    checkBufferSize(bufferSize);
     this.bufferSize = bufferSize;
   }
 
@@ -301,4 +315,21 @@ public abstract class BufferedIndexInput extends IndexInput {
     }
   }
   
+  /**
+   * Returns default buffer sizes for the given {@link IOContext}
+   */
+  public static int bufferSize(IOContext context) {
+    switch (context.context) {
+    case DEFAULT:
+    case FLUSH:
+    case READ:
+      return BUFFER_SIZE;
+    case MERGE:
+      return MERGE_BUFFER_SIZE;
+    default:
+      assert false : "unknown IOContext " + context.context;
+      return BUFFER_SIZE;
+    }
+  }
+  
 }

lucene/src/java/org/apache/lucene/store/FlushInfo.java

@@ -42,4 +42,36 @@ public class FlushInfo {
     this.estimatedSegmentSize = estimatedSegmentSize;
   }
 
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result
+        + (int) (estimatedSegmentSize ^ (estimatedSegmentSize >>> 32));
+    result = prime * result + numDocs;
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    FlushInfo other = (FlushInfo) obj;
+    if (estimatedSegmentSize != other.estimatedSegmentSize)
+      return false;
+    if (numDocs != other.numDocs)
+      return false;
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    return "FlushInfo [numDocs=" + numDocs + ", estimatedSegmentSize="
+        + estimatedSegmentSize + "]";
+  }
+
 }

lucene/src/java/org/apache/lucene/store/IOContext.java

@@ -86,4 +86,47 @@ public class IOContext {
     this.flushInfo = null;
   }
 
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result + ((context == null) ? 0 : context.hashCode());
+    result = prime * result + ((flushInfo == null) ? 0 : flushInfo.hashCode());
+    result = prime * result + ((mergeInfo == null) ? 0 : mergeInfo.hashCode());
+    result = prime * result + (readOnce ? 1231 : 1237);
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    IOContext other = (IOContext) obj;
+    if (context != other.context)
+      return false;
+    if (flushInfo == null) {
+      if (other.flushInfo != null)
+        return false;
+    } else if (!flushInfo.equals(other.flushInfo))
+      return false;
+    if (mergeInfo == null) {
+      if (other.mergeInfo != null)
+        return false;
+    } else if (!mergeInfo.equals(other.mergeInfo))
+      return false;
+    if (readOnce != other.readOnce)
+      return false;
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    return "IOContext [context=" + context + ", mergeInfo=" + mergeInfo
+        + ", flushInfo=" + flushInfo + ", readOnce=" + readOnce + "]";
+  }
+
 }

lucene/src/java/org/apache/lucene/store/MergeInfo.java

@@ -23,13 +23,13 @@ package org.apache.lucene.store;
 
 public class MergeInfo {
   
-  public int totalDocCount;
+  public final int totalDocCount;
   
-  public long estimatedMergeBytes;  // used by IndexWriter
+  public final long estimatedMergeBytes;
   
-  boolean isExternal;               // used by IndexWriter
+  public final boolean isExternal;
   
-  boolean optimize;                 // used by IndexWriter
+  public final boolean optimize;
   
 
   /**
@@ -46,4 +46,44 @@ public class MergeInfo {
     this.isExternal = isExternal;
     this.optimize = optimize;
   }
+
+
+  @Override
+  public int hashCode() {
+    final int prime = 31;
+    int result = 1;
+    result = prime * result
+        + (int) (estimatedMergeBytes ^ (estimatedMergeBytes >>> 32));
+    result = prime * result + (isExternal ? 1231 : 1237);
+    result = prime * result + (optimize ? 1231 : 1237);
+    result = prime * result + totalDocCount;
+    return result;
+  }
+
+  @Override
+  public boolean equals(Object obj) {
+    if (this == obj)
+      return true;
+    if (obj == null)
+      return false;
+    if (getClass() != obj.getClass())
+      return false;
+    MergeInfo other = (MergeInfo) obj;
+    if (estimatedMergeBytes != other.estimatedMergeBytes)
+      return false;
+    if (isExternal != other.isExternal)
+      return false;
+    if (optimize != other.optimize)
+      return false;
+    if (totalDocCount != other.totalDocCount)
+      return false;
+    return true;
+  }
+
+  @Override
+  public String toString() {
+    return "MergeInfo [totalDocCount=" + totalDocCount
+        + ", estimatedMergeBytes=" + estimatedMergeBytes + ", isExternal="
+        + isExternal + ", optimize=" + optimize + "]";
+  }
 }

lucene/src/java/org/apache/lucene/store/SimpleFSDirectory.java

@@ -21,6 +21,8 @@ import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 
+import org.apache.lucene.store.IOContext.Context;
+
 
 /** A straightforward implementation of {@link FSDirectory}
  *  using java.io.RandomAccessFile.  However, this class has
@@ -87,8 +89,7 @@ public class SimpleFSDirectory extends FSDirectory {
     protected final int chunkSize;
     
     public SimpleFSIndexInput(File path, IOContext context, int chunkSize) throws IOException {
-      //nocommit Use IOContext to decide bufferSize instead of BufferedIndexInput.BUFFER_SIZE
-      super(BufferedIndexInput.BUFFER_SIZE);
+      super(context);
       file = new Descriptor(path, "r");
       this.chunkSize = chunkSize;
     }

lucene/src/test-framework/org/apache/lucene/index/codecs/preflexrw/TermInfosWriter.java

@@ -124,7 +124,6 @@ final class TermInfosWriter implements Closeable {
     indexInterval = interval;
     fieldInfos = fis;
     isIndex = isi;
-    // nocommit pass IOContext in via ctor
     output = directory.createOutput(IndexFileNames.segmentFileName(segment, "",
         (isIndex ? PreFlexCodec.TERMS_INDEX_EXTENSION
             : PreFlexCodec.TERMS_EXTENSION)), IOContext.DEFAULT);

lucene/src/test/org/apache/lucene/index/TestCompoundFile.java

@@ -321,14 +321,15 @@ public class TestCompoundFile extends LuceneTestCase
     private void demo_FSIndexInputBug(Directory fsdir, String file)
     throws IOException
     {
+        // IOContext triggers different buffer sizes so we use default here
         // Setup the test file - we need more than 1024 bytes
-        IndexOutput os = fsdir.createOutput(file, newIOContext(random));
+        IndexOutput os = fsdir.createOutput(file, IOContext.DEFAULT);
         for(int i=0; i<2000; i++) {
             os.writeByte((byte) i);
         }
         os.close();
 
-        IndexInput in = fsdir.openInput(file, newIOContext(random));
+        IndexInput in = fsdir.openInput(file, IOContext.DEFAULT);
 
         // This read primes the buffer in IndexInput
         in.readByte();