Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-2898: fix CMS throttling to be independent of number of incoming producer threads; some defensive concurrency fixes for SegmentInfo 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1065059 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2011-01-29 16:22:59 +00:00
parent cbf8d878f7
commit 5342d8676d
2 changed files with 41 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

48
lucene/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
View File

@ -308,10 +308,31 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
// pending merges, until it's empty:
while (true) {
synchronized(this) {
long startStallTime = 0;
while (mergeThreadCount() >= 1+maxMergeCount) {
startStallTime = System.currentTimeMillis();
if (verbose()) {
message(" too many merges; stalling...");
}
try {
wait();
} catch (InterruptedException ie) {
throw new ThreadInterruptedException(ie);
}
}
if (verbose()) {
if (startStallTime != 0) {
message(" stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
}
}
}
// TODO: we could be careful about which merges to do in
// the BG (eg maybe the "biggest" ones) vs FG, which
// merges to do first (the easiest ones?), etc.
MergePolicy.OneMerge merge = writer.getNextMerge();
if (merge == null) {
if (verbose())
@ -326,32 +347,11 @@ public class ConcurrentMergeScheduler extends MergeScheduler {
boolean success = false;
try {
synchronized(this) {
final MergeThread merger;
long startStallTime = 0;
while (mergeThreadCount() >= maxMergeCount) {
startStallTime = System.currentTimeMillis();
if (verbose()) {
message(" too many merges; stalling...");
}
try {
wait();
} catch (InterruptedException ie) {
throw new ThreadInterruptedException(ie);
}
}
if (verbose()) {
if (startStallTime != 0) {
message(" stalled for " + (System.currentTimeMillis()-startStallTime) + " msec");
}
message(" consider merge " + merge.segString(dir));
}
assert mergeThreadCount() < maxMergeCount;
message(" consider merge " + merge.segString(dir));
// OK to spawn a new merge thread to handle this
// merge:
merger = getMergeThread(writer, merge);
final MergeThread merger = getMergeThread(writer, merge);
mergeThreads.add(merger);
if (verbose()) {
message(" launch new thread [" + merger.getName() + "]");

27
lucene/src/java/org/apache/lucene/index/SegmentInfo.java
View File

@ -66,11 +66,11 @@ public final class SegmentInfo {
private boolean isCompoundFile;
private List<String> files; // cached list of files that this segment uses
private volatile List<String> files; // cached list of files that this segment uses
// in the Directory
private long sizeInBytesNoStore = -1; // total byte size of all but the store files (computed on demand)
private long sizeInBytesWithStore = -1; // total byte size of all of our files (computed on demand)
private volatile long sizeInBytesNoStore = -1; // total byte size of all but the store files (computed on demand)
private volatile long sizeInBytesWithStore = -1; // total byte size of all of our files (computed on demand)
private int docStoreOffset; // if this segment shares stored fields & vectors, this
// offset is where in that file this segment's docs begin
@ -241,24 +241,31 @@ public final class SegmentInfo {
*/
public long sizeInBytes(boolean includeDocStores) throws IOException {
if (includeDocStores) {
if (sizeInBytesWithStore != -1) return sizeInBytesWithStore;
sizeInBytesWithStore = 0;
if (sizeInBytesWithStore != -1) {
return sizeInBytesWithStore;
}
long sum = 0;
for (final String fileName : files()) {
// We don't count bytes used by a shared doc store against this segment
// We don't count bytes used by a shared doc store
// against this segment
if (docStoreOffset == -1 || !IndexFileNames.isDocStoreFile(fileName)) {
sizeInBytesWithStore += dir.fileLength(fileName);
sum += dir.fileLength(fileName);
}
}
sizeInBytesWithStore = sum;
return sizeInBytesWithStore;
} else {
if (sizeInBytesNoStore != -1) return sizeInBytesNoStore;
sizeInBytesNoStore = 0;
if (sizeInBytesNoStore != -1) {
return sizeInBytesNoStore;
}
long sum = 0;
for (final String fileName : files()) {
if (IndexFileNames.isDocStoreFile(fileName)) {
continue;
}
sizeInBytesNoStore += dir.fileLength(fileName);
sum += dir.fileLength(fileName);
}
sizeInBytesNoStore = sum;
return sizeInBytesNoStore;
}
}