fix IW infoStream to not include doc store files size when reporting flushed postings size of new segment vs RAM size

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1051058 13f79535-47bb-0310-9956-ffa450edef68

lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java

@@ -56,7 +56,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
   
   @Override
   protected long size(SegmentInfo info) throws IOException {
-    long byteSize = info.sizeInBytes();
+    long byteSize = info.sizeInBytes(true);
     float delRatio = (info.docCount <= 0 ? 0.0f : ((float)info.getDelCount() / (float)info.docCount));
     return (info.docCount <= 0 ?  byteSize : (long)((1.0f - delRatio) * byteSize));
   }

lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java

@@ -26,7 +26,6 @@ import java.text.DecimalFormat;
 import java.util.ArrayList;
 import java.util.List;
 
-import org.apache.lucene.index.codecs.Codec;
 import org.apache.lucene.index.codecs.CodecProvider;
 import org.apache.lucene.store.FSDirectory;
 
@@ -108,7 +107,7 @@ public class IndexSplitter {
     DecimalFormat formatter = new DecimalFormat("###,###.###");
     for (int x = 0; x < infos.size(); x++) {
       SegmentInfo info = infos.info(x);
-      String sizeStr = formatter.format(info.sizeInBytes());
+      String sizeStr = formatter.format(info.sizeInBytes(true));
       System.out.println(info.name + " " + sizeStr);
     }
   }

lucene/src/java/org/apache/lucene/index/CheckIndex.java

@@ -417,8 +417,8 @@ public class CheckIndex {
         segInfoStat.hasProx = info.getHasProx();
         msg("    numFiles=" + info.files().size());
         segInfoStat.numFiles = info.files().size();
-        msg("    size (MB)=" + nf.format(info.sizeInBytes()/(1024.*1024.)));
+        segInfoStat.sizeMB = info.sizeInBytes(true)/(1024.*1024.);
-        segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
+        msg("    size (MB)=" + nf.format(segInfoStat.sizeMB));
         Map<String,String> diagnostics = info.getDiagnostics();
         segInfoStat.diagnostics = diagnostics;
         if (diagnostics.size() > 0) {

lucene/src/java/org/apache/lucene/index/DocumentsWriter.java

@@ -622,11 +622,13 @@ final class DocumentsWriter {
 
       if (infoStream != null) {
         message("flush: segment=" + newSegment);
-        final long newSegmentSize = newSegment.sizeInBytes();
+        final long newSegmentSizeNoStore = newSegment.sizeInBytes(false);
+        final long newSegmentSize = newSegment.sizeInBytes(true);
         message("  ramUsed=" + nf.format(startNumBytesUsed / 1024. / 1024.) + " MB" +
-            " newFlushedSize=" + nf.format(newSegmentSize / 1024 / 1024) + " MB" +
+                " newFlushedSize=" + nf.format(newSegmentSize / 1024 / 1024) + " MB" +
-            " docs/MB=" + nf.format(numDocs / (newSegmentSize / 1024. / 1024.)) +
+                " (" + nf.format(newSegmentSizeNoStore / 1024 / 1024) + " MB w/o doc stores)" +
-            " new/old=" + nf.format(100.0 * newSegmentSize / startNumBytesUsed) + "%");
+                " docs/MB=" + nf.format(numDocs / (newSegmentSize / 1024. / 1024.)) +
+                " new/old=" + nf.format(100.0 * newSegmentSizeNoStore / startNumBytesUsed) + "%");
       }
 
       success = true;

lucene/src/java/org/apache/lucene/index/LogMergePolicy.java

@@ -184,7 +184,7 @@ public abstract class LogMergePolicy extends MergePolicy {
   }
   
   protected long sizeBytes(SegmentInfo info) throws IOException {
-    long byteSize = info.sizeInBytes();
+    long byteSize = info.sizeInBytes(true);
     if (calibrateSizeByDeletes) {
       int delCount = writer.get().numDeletedDocs(info);
       double delRatio = (info.docCount <= 0 ? 0.0f : ((float)delCount / (float)info.docCount));

lucene/src/java/org/apache/lucene/index/MergePolicy.java

@@ -165,7 +165,7 @@ public abstract class MergePolicy implements java.io.Closeable {
     public long totalBytesSize() throws IOException {
       long total = 0;
       for (SegmentInfo info : segments) {
-        total += info.sizeInBytes();
+        total += info.sizeInBytes(true);
       }
       return total;
     }

lucene/src/java/org/apache/lucene/index/SegmentInfo.java

@@ -220,13 +220,16 @@ public final class SegmentInfo {
   
   /** Returns total size in bytes of all of files used by
    *  this segment. */
-  public long sizeInBytes() throws IOException {
+  public long sizeInBytes(boolean includeDocStores) throws IOException {
     if (sizeInBytes == -1) {
       List<String> files = files();
       final int size = files.size();
       sizeInBytes = 0;
       for(int i=0;i<size;i++) {
         final String fileName = files.get(i);
+        if (!includeDocStores && IndexFileNames.isDocStoreFile(fileName)) {
+          continue;
+        }
         // We don't count bytes used by a shared doc store
         // against this segment:
         if (docStoreOffset == -1 || !IndexFileNames.isDocStoreFile(fileName))

lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java

@@ -59,7 +59,7 @@ public class TestSizeBoundedOptimize extends LuceneTestCase {
 
     SegmentInfos sis = new SegmentInfos();
     sis.read(dir);
-    double min = sis.info(0).sizeInBytes();
+    double min = sis.info(0).sizeInBytes(true);
 
     conf = newWriterConfig();
     LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();