LUCENE-431: RAMInputStream and RAMOutputStream subclass IndexInput and IndexOutput directly now

git-svn-id: https://svn.apache.org/repos/asf/lucene/java/trunk@529749 13f79535-47bb-0310-9956-ffa450edef68

CHANGES.txt

@@ -130,6 +130,10 @@ Optimizations
     of SegmentTermPositions instead of SegmentTermDocs without additional costs.
     (Michael Busch)
 
+ 2. LUCENE-431: RAMInputStream and RAMOutputStream extend IndexInput and
+    IndexOutput directly now. This avoids further buffering and thus avoids 
+    unneccessary array copies. (Michael Busch)
+
 Documentation:
  1. LUCENE 791 && INFRA-1173: Infrastructure moved the Wiki to http://wiki.apache.org/lucene-java/   Updated the links in the docs and wherever else I found references.  (Grant Ingersoll, Joe Schaefer)
 

src/java/org/apache/lucene/store/RAMInputStream.java

@@ -1,5 +1,7 @@
 package org.apache.lucene.store;
 
+import java.io.IOException;
+
 /**
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -19,46 +21,90 @@ package org.apache.lucene.store;
 
 /**
  * A memory-resident {@link IndexInput} implementation.
- *
+ * 
  * @version $Id$
  */
 
-class RAMInputStream extends BufferedIndexInput implements Cloneable {
+class RAMInputStream extends IndexInput implements Cloneable {
+  static final int BUFFER_SIZE = BufferedIndexOutput.BUFFER_SIZE;
+
   private RAMFile file;
-  private long pointer = 0;
   private long length;
 
+  private byte[] currentBuffer;
+  private int currentBufferIndex;
+  
+  private int bufferPosition;
+  private long bufferStart;
+  private int bufferLength;
+
   public RAMInputStream(RAMFile f) {
     file = f;
     length = file.length;
-  }
 
-  public void readInternal(byte[] dest, int destOffset, int len) {
-    int remainder = len;
-    long start = pointer;
-    while (remainder != 0) {
-      int bufferNumber = (int)(start/BUFFER_SIZE);
-      int bufferOffset = (int)(start%BUFFER_SIZE);
-      int bytesInBuffer = BUFFER_SIZE - bufferOffset;
-      int bytesToCopy = bytesInBuffer >= remainder ? remainder : bytesInBuffer;
-      byte[] buffer = (byte[])file.buffers.get(bufferNumber);
-      System.arraycopy(buffer, bufferOffset, dest, destOffset, bytesToCopy);
-      destOffset += bytesToCopy;
-      start += bytesToCopy;
-      remainder -= bytesToCopy;
-    }
-    pointer += len;
+    // make sure that we switch to the
+    // first needed buffer lazily
+    currentBufferIndex = -1;
+    currentBuffer = null;
   }
 
   public void close() {
-  }
-
-  public void seekInternal(long pos) {
-    pointer = pos;
+    // nothing to do here
   }
 
   public long length() {
     return length;
   }
 
+  public byte readByte() throws IOException {
+    if (bufferPosition >= bufferLength) {
+      currentBufferIndex++;
+      switchCurrentBuffer();
+    }
+    return currentBuffer[bufferPosition++];
+  }
+
+  public void readBytes(byte[] b, int offset, int len) throws IOException {
+    while (len > 0) {
+      if (bufferPosition >= bufferLength) {
+        currentBufferIndex++;
+        switchCurrentBuffer();
+      }
+
+      int remainInBuffer = bufferLength - bufferPosition;
+      int bytesToCopy = len < remainInBuffer ? len : remainInBuffer;
+      System.arraycopy(currentBuffer, bufferPosition, b, offset, bytesToCopy);
+      offset += bytesToCopy;
+      len -= bytesToCopy;
+      bufferPosition += bytesToCopy;
+    }
+  }
+
+  private final void switchCurrentBuffer() throws IOException {
+    if (currentBufferIndex >= file.buffers.size()) {
+      // end of file reached, no more buffers left
+      throw new IOException("Read past EOF");
+    } else {
+      currentBuffer = (byte[]) file.buffers.get(currentBufferIndex);
+      bufferPosition = 0;
+      bufferStart = BUFFER_SIZE * currentBufferIndex;
+      bufferLength = (int) (length - bufferStart);
+      if (bufferLength > BUFFER_SIZE) {
+        bufferLength = BUFFER_SIZE;
+      }
+    }
+  }
+
+  public long getFilePointer() {
+    return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition;
+  }
+
+  public void seek(long pos) throws IOException {
+    long bufferStart = currentBufferIndex * BUFFER_SIZE;
+    if (pos < bufferStart || pos >= bufferStart + BUFFER_SIZE) {
+      currentBufferIndex = (int) (pos / BUFFER_SIZE);
+      switchCurrentBuffer();
+    }
+    bufferPosition = (int) (pos % BUFFER_SIZE);
+  }
 }

src/java/org/apache/lucene/store/RAMOutputStream.java

@@ -21,13 +21,21 @@ import java.io.IOException;
 
 /**
  * A memory-resident {@link IndexOutput} implementation.
- *
+ * 
  * @version $Id$
  */
 
-public class RAMOutputStream extends BufferedIndexOutput {
+public class RAMOutputStream extends IndexOutput {
+  static final int BUFFER_SIZE = BufferedIndexOutput.BUFFER_SIZE;
+
   private RAMFile file;
-  private long pointer = 0;
+
+  private byte[] currentBuffer;
+  private int currentBufferIndex;
+  
+  private int bufferPosition;
+  private long bufferStart;
+  private int bufferLength;
 
   /** Construct an empty output buffer. */
   public RAMOutputStream() {
@@ -36,6 +44,11 @@ public class RAMOutputStream extends BufferedIndexOutput {
 
   RAMOutputStream(RAMFile f) {
     file = f;
+
+    // make sure that we switch to the
+    // first needed buffer lazily
+    currentBufferIndex = -1;
+    currentBuffer = null;
   }
 
   /** Copy the current contents of this buffer to the named output. */
@@ -66,41 +79,74 @@ public class RAMOutputStream extends BufferedIndexOutput {
     file.setLength(0);
   }
 
-  public void flushBuffer(byte[] src, int offset, int len) throws IOException {
-    byte[] buffer;
-    int bufferPos = 0;
-    while (bufferPos != len) {
-      int bufferNumber = (int)(pointer/BUFFER_SIZE);
-      int bufferOffset = (int)(pointer%BUFFER_SIZE);
-      int bytesInBuffer = BUFFER_SIZE - bufferOffset;
-      int remainInSrcBuffer = len - bufferPos;
-      int bytesToCopy = bytesInBuffer >= remainInSrcBuffer ? remainInSrcBuffer : bytesInBuffer;
-
-      if (bufferNumber == file.buffers.size())
-        buffer = file.addBuffer(BUFFER_SIZE);
-      else
-        buffer = (byte[]) file.buffers.get(bufferNumber);
-
-      System.arraycopy(src, offset + bufferPos, buffer, bufferOffset, bytesToCopy);
-      bufferPos += bytesToCopy;
-      pointer += bytesToCopy;
-    }
-
-    if (pointer > file.length)
-      file.setLength(pointer);
-
-    file.setLastModified(System.currentTimeMillis());
-  }
-
   public void close() throws IOException {
-    super.close();
+    flush();
   }
 
   public void seek(long pos) throws IOException {
-    super.seek(pos);
-    pointer = pos;
+    // set the file length in case we seek back
+    // and flush() has not been called yet
+    setFileLength();
+    if (pos < bufferStart || pos >= bufferStart + bufferLength) {
+      currentBufferIndex = (int) (pos / BUFFER_SIZE);
+      switchCurrentBuffer();
+    }
+
+    bufferPosition = (int) (pos % BUFFER_SIZE);
   }
+
   public long length() {
     return file.length;
   }
+
+  public void writeByte(byte b) throws IOException {
+    if (bufferPosition == bufferLength) {
+      currentBufferIndex++;
+      switchCurrentBuffer();
+    }
+    currentBuffer[bufferPosition++] = b;
+  }
+
+  public void writeBytes(byte[] b, int offset, int len) throws IOException {
+    while (len > 0) {
+      if (bufferPosition ==  bufferLength) {
+        currentBufferIndex++;
+        switchCurrentBuffer();
+      }
+
+      int remainInBuffer = currentBuffer.length - bufferPosition;
+      int bytesToCopy = len < remainInBuffer ? len : remainInBuffer;
+      System.arraycopy(b, offset, currentBuffer, bufferPosition, bytesToCopy);
+      offset += bytesToCopy;
+      len -= bytesToCopy;
+      bufferPosition += bytesToCopy;
+    }
+  }
+
+  private final void switchCurrentBuffer() throws IOException {
+    if (currentBufferIndex == file.buffers.size()) {
+      currentBuffer = file.addBuffer(BUFFER_SIZE);
+    } else {
+      currentBuffer = (byte[]) file.buffers.get(currentBufferIndex);
+    }
+    bufferPosition = 0;
+    bufferStart = BUFFER_SIZE * currentBufferIndex;
+    bufferLength = currentBuffer.length;
+  }
+
+  private void setFileLength() {
+    long pointer = bufferStart + bufferPosition;
+    if (pointer > file.length) {
+      file.setLength(pointer);
+    }
+  }
+
+  public void flush() throws IOException {
+    file.setLastModified(System.currentTimeMillis());
+    setFileLength();
+  }
+
+  public long getFilePointer() {
+    return currentBufferIndex < 0 ? 0 : bufferStart + bufferPosition;
+  }
 }

src/test/org/apache/lucene/store/MockRAMOutputStream.java

@@ -30,6 +30,8 @@ import java.util.Iterator;
 public class MockRAMOutputStream extends RAMOutputStream {
   private MockRAMDirectory dir;
   private boolean first=true;
+  
+  byte[] singleByte = new byte[1];
 
   /** Construct an empty output buffer. */
   public MockRAMOutputStream(MockRAMDirectory dir, RAMFile f) {
@@ -48,7 +50,12 @@ public class MockRAMOutputStream extends RAMOutputStream {
     }
   }
 
-  public void flushBuffer(byte[] src, int offset, int len) throws IOException {
+  public void writeByte(byte b) throws IOException {
+    singleByte[0] = b;
+    writeBytes(singleByte, 0, 1);
+  }
+  
+    public void writeBytes(byte[] b, int offset, int len) throws IOException {
     long freeSpace = dir.maxSize - dir.sizeInBytes();
     long realUsage = 0;
 
@@ -63,14 +70,14 @@ public class MockRAMOutputStream extends RAMOutputStream {
     if (dir.maxSize != 0 && freeSpace <= len) {
       if (freeSpace > 0 && freeSpace < len) {
         realUsage += freeSpace;
-        super.flushBuffer(src, offset, (int) freeSpace);
+        super.writeBytes(b, offset, (int) freeSpace);
       }
       if (realUsage > dir.maxUsedSize) {
         dir.maxUsedSize = realUsage;
       }
       throw new IOException("fake disk full at " + dir.getRecomputedActualSizeInBytes() + " bytes");
     } else {
-      super.flushBuffer(src, offset, len);
+      super.writeBytes(b, offset, len);
     }
 
     if (first) {