Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

fix IW infoStream to not include doc store files size when reporting flushed postings size of new segment vs RAM size 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1051058 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2010-12-20 10:53:57 +00:00
parent 534e34dc51
commit 5b3250dc37
8 changed files with 17 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lucene/contrib/misc/src/java/org/apache/lucene/index/BalancedSegmentMergePolicy.java
View File

@ -56,7 +56,7 @@ public class BalancedSegmentMergePolicy extends LogByteSizeMergePolicy {
@Override
protected long size(SegmentInfo info) throws IOException {
long byteSize = info.sizeInBytes();
long byteSize = info.sizeInBytes(true);
float delRatio = (info.docCount <= 0 ? 0.0f : ((float)info.getDelCount() / (float)info.docCount));
return (info.docCount <= 0 ? byteSize : (long)((1.0f - delRatio) * byteSize));
}

3
lucene/contrib/misc/src/java/org/apache/lucene/index/IndexSplitter.java
View File

@ -26,7 +26,6 @@ import java.text.DecimalFormat;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.CodecProvider;
import org.apache.lucene.store.FSDirectory;
@ -108,7 +107,7 @@ public class IndexSplitter {
DecimalFormat formatter = new DecimalFormat("###,###.###");
for (int x = 0; x < infos.size(); x++) {
SegmentInfo info = infos.info(x);
String sizeStr = formatter.format(info.sizeInBytes());
String sizeStr = formatter.format(info.sizeInBytes(true));
System.out.println(info.name + " " + sizeStr);
}
}

4
lucene/src/java/org/apache/lucene/index/CheckIndex.java
View File

@ -417,8 +417,8 @@ public class CheckIndex {
segInfoStat.hasProx = info.getHasProx();
msg(" numFiles=" + info.files().size());
segInfoStat.numFiles = info.files().size();
msg(" size (MB)=" + nf.format(info.sizeInBytes()/(1024.*1024.)));
segInfoStat.sizeMB = info.sizeInBytes()/(1024.*1024.);
segInfoStat.sizeMB = info.sizeInBytes(true)/(1024.*1024.);
msg(" size (MB)=" + nf.format(segInfoStat.sizeMB));
Map<String,String> diagnostics = info.getDiagnostics();
segInfoStat.diagnostics = diagnostics;
if (diagnostics.size() > 0) {

10
lucene/src/java/org/apache/lucene/index/DocumentsWriter.java
View File

@ -622,11 +622,13 @@ final class DocumentsWriter {
if (infoStream != null) {
message("flush: segment=" + newSegment);
final long newSegmentSize = newSegment.sizeInBytes();
final long newSegmentSizeNoStore = newSegment.sizeInBytes(false);
final long newSegmentSize = newSegment.sizeInBytes(true);
message(" ramUsed=" + nf.format(startNumBytesUsed / 1024. / 1024.) + " MB" +
" newFlushedSize=" + nf.format(newSegmentSize / 1024 / 1024) + " MB" +
" docs/MB=" + nf.format(numDocs / (newSegmentSize / 1024. / 1024.)) +
" new/old=" + nf.format(100.0 * newSegmentSize / startNumBytesUsed) + "%");
" newFlushedSize=" + nf.format(newSegmentSize / 1024 / 1024) + " MB" +
" (" + nf.format(newSegmentSizeNoStore / 1024 / 1024) + " MB w/o doc stores)" +
" docs/MB=" + nf.format(numDocs / (newSegmentSize / 1024. / 1024.)) +
" new/old=" + nf.format(100.0 * newSegmentSizeNoStore / startNumBytesUsed) + "%");
}
success = true;

2
lucene/src/java/org/apache/lucene/index/LogMergePolicy.java
View File

@ -184,7 +184,7 @@ public abstract class LogMergePolicy extends MergePolicy {
}
protected long sizeBytes(SegmentInfo info) throws IOException {
long byteSize = info.sizeInBytes();
long byteSize = info.sizeInBytes(true);
if (calibrateSizeByDeletes) {
int delCount = writer.get().numDeletedDocs(info);
double delRatio = (info.docCount <= 0 ? 0.0f : ((float)delCount / (float)info.docCount));

2
lucene/src/java/org/apache/lucene/index/MergePolicy.java
View File

@ -165,7 +165,7 @@ public abstract class MergePolicy implements java.io.Closeable {
public long totalBytesSize() throws IOException {
long total = 0;
for (SegmentInfo info : segments) {
total += info.sizeInBytes();
total += info.sizeInBytes(true);
}
return total;
}

5
lucene/src/java/org/apache/lucene/index/SegmentInfo.java
View File

@ -220,13 +220,16 @@ public final class SegmentInfo {
/** Returns total size in bytes of all of files used by
* this segment. */
public long sizeInBytes() throws IOException {
public long sizeInBytes(boolean includeDocStores) throws IOException {
if (sizeInBytes == -1) {
List<String> files = files();
final int size = files.size();
sizeInBytes = 0;
for(int i=0;i<size;i++) {
final String fileName = files.get(i);
if (!includeDocStores && IndexFileNames.isDocStoreFile(fileName)) {
continue;
}
// We don't count bytes used by a shared doc store
// against this segment:
if (docStoreOffset == -1 || !IndexFileNames.isDocStoreFile(fileName))

2
lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
View File

@ -59,7 +59,7 @@ public class TestSizeBoundedOptimize extends LuceneTestCase {
SegmentInfos sis = new SegmentInfos();
sis.read(dir);
double min = sis.info(0).sizeInBytes();
double min = sis.info(0).sizeInBytes(true);
conf = newWriterConfig();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();