LUCENE-3892: decouple level 0 skipInterval from level 1+

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/pforcodec_3892@1370055 13f79535-47bb-0310-9956-ffa450edef68

lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListReader.java

@@ -65,24 +65,31 @@ public abstract class MultiLevelSkipListReader {
   private long lastChildPointer;      // childPointer of last read skip entry with docId <= target
   
   private boolean inputIsBuffered;
-  
-  public MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) {
+  private final int skipMultiplier;
+
+  // nocommit made protected from public
+  protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval, int skipMultiplier) {
     this.skipStream = new IndexInput[maxSkipLevels];
     this.skipPointer = new long[maxSkipLevels];
     this.childPointer = new long[maxSkipLevels];
     this.numSkipped = new int[maxSkipLevels];
     this.maxNumberOfSkipLevels = maxSkipLevels;
     this.skipInterval = new int[maxSkipLevels];
+    this.skipMultiplier = skipMultiplier;
     this.skipStream [0]= skipStream;
     this.inputIsBuffered = (skipStream instanceof BufferedIndexInput);
     this.skipInterval[0] = skipInterval;
     for (int i = 1; i < maxSkipLevels; i++) {
       // cache skip intervals
-      this.skipInterval[i] = this.skipInterval[i - 1] * skipInterval;
+      this.skipInterval[i] = this.skipInterval[i - 1] * skipMultiplier;
     }
     skipDoc = new int[maxSkipLevels];
   }
 
+  // skipMultiplier and skipInterval are the same:
+  protected MultiLevelSkipListReader(IndexInput skipStream, int maxSkipLevels, int skipInterval) {
+    this(skipStream, maxSkipLevels, skipInterval, skipInterval);
+  }
   
   /** Returns the id of the doc to which the last call of {@link #skipTo(int)}
    *  has skipped.  */
@@ -157,7 +164,7 @@ public abstract class MultiLevelSkipListReader {
     numSkipped[level] = numSkipped[level + 1] - skipInterval[level + 1];
     skipDoc[level] = lastDoc;
     if (level > 0) {
-        childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1];
+      childPointer[level] = skipStream[level].readVLong() + skipPointer[level - 1];
     }
   }
 
@@ -187,7 +194,12 @@ public abstract class MultiLevelSkipListReader {
   
   /** Loads the skip levels  */
   private void loadSkipLevels() throws IOException {
-    numberOfSkipLevels = MathUtil.log(docCount, skipInterval[0]);
+    if (docCount <= skipInterval[0]) {
+      numberOfSkipLevels = 1;
+    } else {
+      numberOfSkipLevels = 1+MathUtil.log(docCount/skipInterval[0], skipMultiplier);
+    }
+
     if (numberOfSkipLevels > maxNumberOfSkipLevels) {
       numberOfSkipLevels = maxNumberOfSkipLevels;
     }

lucene/core/src/java/org/apache/lucene/codecs/MultiLevelSkipListWriter.java

@@ -54,15 +54,23 @@ public abstract class MultiLevelSkipListWriter {
   
   // the skip interval in the list with level = 0
   private int skipInterval;
+
+  // skipInterval used for level > 0
+  private int skipMultiplier;
   
   // for every skip level a different buffer is used 
   private RAMOutputStream[] skipBuffer;
 
-  protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) {
+  protected MultiLevelSkipListWriter(int skipInterval, int skipMultiplier, int maxSkipLevels, int df) {
     this.skipInterval = skipInterval;
+    this.skipMultiplier = skipMultiplier;
     
     // calculate the maximum number of skip levels for this document frequency
-    numberOfSkipLevels = MathUtil.log(df, skipInterval);
+    if (df <= skipInterval) {
+      numberOfSkipLevels = 1;
+    } else {
+      numberOfSkipLevels = 1+MathUtil.log(df/skipInterval, skipMultiplier);
+    }
     
     // make sure it does not exceed maxSkipLevels
     if (numberOfSkipLevels > maxSkipLevels) {
@@ -70,6 +78,11 @@ public abstract class MultiLevelSkipListWriter {
     }
   }
   
+  // skipMultiplier and skipInterval are the same:
+  protected MultiLevelSkipListWriter(int skipInterval, int maxSkipLevels, int df) {
+    this(skipInterval, skipInterval, maxSkipLevels, df);
+  }
+
   protected void init() {
     skipBuffer = new RAMOutputStream[numberOfSkipLevels];
     for (int i = 0; i < numberOfSkipLevels; i++) {
@@ -95,7 +108,7 @@ public abstract class MultiLevelSkipListWriter {
    * @param skipBuffer the skip buffer to write to
    */
   protected abstract void writeSkipData(int level, IndexOutput skipBuffer) throws IOException;
-  
+
   /**
    * Writes the current skip data to the buffers. The current document frequency determines
    * the max level is skip data is to be written to. 
@@ -104,11 +117,15 @@ public abstract class MultiLevelSkipListWriter {
    * @throws IOException
    */
   public void bufferSkip(int df) throws IOException {
-    int numLevels;
+
+    assert df % skipInterval == 0;
+    int numLevels = 1;
+    df /= skipInterval;
    
     // determine max level
-    for (numLevels = 0; (df % skipInterval) == 0 && numLevels < numberOfSkipLevels; df /= skipInterval) {
+    while ((df % skipMultiplier) == 0 && numLevels < numberOfSkipLevels) {
       numLevels++;
+      df /= skipMultiplier;
     }
     
     long childPointer = 0;
@@ -150,5 +167,4 @@ public abstract class MultiLevelSkipListWriter {
     
     return skipPointer;
   }
-
 }

lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipReader.java

@@ -69,7 +69,7 @@ final class BlockSkipReader extends MultiLevelSkipListReader {
   private int lastPosBufferUpto;
 
   public BlockSkipReader(IndexInput skipStream, int maxSkipLevels, int blockSize, boolean hasPos, boolean hasOffsets, boolean hasPayloads) {
-    super(skipStream, maxSkipLevels, blockSize);
+    super(skipStream, maxSkipLevels, blockSize, 4);
     this.blockSize = blockSize;
     docPointer = new long[maxSkipLevels];
     if (hasPos) {

lucene/core/src/java/org/apache/lucene/codecs/block/BlockSkipWriter.java

@@ -72,7 +72,9 @@ final class BlockSkipWriter extends MultiLevelSkipListWriter {
   private boolean fieldHasPayloads;
 
   public BlockSkipWriter(int maxSkipLevels, int blockSize, int docCount, IndexOutput docOut, IndexOutput posOut, IndexOutput payOut) {
-    super(blockSize, maxSkipLevels, docCount);
+    // nocommit figure out what skipMultiplier is best (4 is
+    // total guess):
+    super(blockSize, 4, maxSkipLevels, docCount);
     this.docOut = docOut;
     this.posOut = posOut;
     this.payOut = payOut;