Apache
/
lucene
mirror of https://github.com/apache/lucene.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

LUCENE-3661: remove MutableBits.count() 

git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/branches/lucene3661@1237057 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Michael McCandless 2012-01-28 13:23:40 +00:00
parent e1a808d489
commit d79594dd14
8 changed files with 100 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
lucene/src/java/org/apache/lucene/codecs/lucene40/Lucene40LiveDocsFormat.java
View File

@ -26,14 +26,20 @@ public class Lucene40LiveDocsFormat extends LiveDocsFormat {
@Override
public Bits readLiveDocs(Directory dir, SegmentInfo info, IOContext context) throws IOException {
String filename = IndexFileNames.fileNameFromGeneration(info.name, DELETES_EXTENSION, info.getDelGen());
return new BitVector(dir, filename, context);
final BitVector liveDocs = new BitVector(dir, filename, context);
assert liveDocs.count() == info.docCount - info.getDelCount();
assert liveDocs.length() == info.docCount;
return liveDocs;
}
@Override
public void writeLiveDocs(MutableBits bits, Directory dir, SegmentInfo info, IOContext context) throws IOException {
// nocommit: this api is ugly...
String filename = IndexFileNames.fileNameFromGeneration(info.name, DELETES_EXTENSION, info.getDelGen());
((BitVector)bits).write(dir, filename, context);
final BitVector liveDocs = (BitVector) bits;
assert liveDocs.count() == info.docCount - info.getDelCount();
assert liveDocs.length() == info.docCount;
liveDocs.write(dir, filename, context);
}
@Override

5
lucene/src/java/org/apache/lucene/codecs/simpletext/SimpleTextLiveDocsFormat.java
View File

@ -168,11 +168,6 @@ public class SimpleTextLiveDocsFormat extends LiveDocsFormat {
bits.clear(bit);
}
@Override
public int count() {
return bits.cardinality();
}
@Override
public SimpleTextBits clone() {
BitSet clonedBits = (BitSet) bits.clone();

2
lucene/src/java/org/apache/lucene/index/DocumentsWriterPerThread.java
View File

@ -477,7 +477,7 @@ public class DocumentsWriterPerThread {
pendingDeletes.terms.clear();
final SegmentInfo newSegment = new SegmentInfo(segment, flushState.numDocs, directory, false, flushState.codec, fieldInfos.asReadOnly());
if (infoStream.isEnabled("DWPT")) {
infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.liveDocs.count())) + " deleted docs");
infoStream.message("DWPT", "new segment has " + (flushState.liveDocs == null ? 0 : (flushState.numDocs - flushState.delCountOnFlush)) + " deleted docs");
infoStream.message("DWPT", "new segment has " + (newSegment.getHasVectors() ? "vectors" : "no vectors"));
infoStream.message("DWPT", "flushedFiles=" + newSegment.files());
infoStream.message("DWPT", "flushed codec=" + newSegment.getCodec());

20
lucene/src/java/org/apache/lucene/index/IndexWriter.java
View File

@ -468,17 +468,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
return reader != null || mergeReader != null;
}
// Called only from assert
private boolean countsMatch() {
if (liveDocs == null) {
assert pendingDeleteCount == 0;
} else {
assert liveDocs.count() == info.docCount - info.getDelCount() - pendingDeleteCount :
"liveDocs.count()=" + liveDocs.count() + " info.docCount=" + info.docCount + " info.delCount=" + info.getDelCount() + " pendingDelCount=" + pendingDeleteCount;
}
return true;
}
// Get reader for searching/deleting
public synchronized SegmentReader getReader(IOContext context) throws IOException {
//System.out.println(" livedocs=" + rld.liveDocs);
@ -559,7 +548,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
getReader(context).decRef();
assert reader != null;
}
assert countsMatch();
shared = true;
if (liveDocs != null) {
return new SegmentReader(reader, liveDocs, info.docCount - info.getDelCount() - pendingDeleteCount);
@ -593,7 +581,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
//System.out.println("getROLiveDocs seg=" + info);
assert Thread.holdsLock(IndexWriter.this);
shared = true;
assert countsMatch();
//if (liveDocs != null) {
//System.out.println(" liveCount=" + liveDocs.count());
//}
@ -612,6 +599,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
// Save in case we need to rollback on failure:
final SegmentInfo sav = (SegmentInfo) info.clone();
info.advanceDelGen();
info.setDelCount(info.getDelCount() + pendingDeleteCount);
// We can write directly to the actual name (vs to a
// .tmp & renaming it) because the file is not live
@ -625,9 +613,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
info.reset(sav);
}
}
assert (info.docCount - liveDocs.count()) == info.getDelCount() + pendingDeleteCount:
"delete count mismatch during commit: seg=" + info + " info.delCount=" + info.getDelCount() + " vs MutableBits=" + (info.docCount-liveDocs.count() + " pendingDelCount=" + pendingDeleteCount);
info.setDelCount(info.getDelCount() + pendingDeleteCount);
pendingDeleteCount = 0;
return true;
} else {
@ -3046,7 +3031,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
// This means this segment received new deletes
// since we started the merge, so we
// must merge them:
final int startDocUpto = docUpto;
for(int j=0;j<docCount;j++) {
if (!prevLiveDocs.get(j)) {
assert !currentLiveDocs.get(j);
@ -3062,8 +3046,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit {
}
}
} else {
assert mergeState.readers != null;
assert mergeState.segmentDocCounts != null;
docUpto += mergeState.segmentDocCounts.get(info);
}
} else if (currentLiveDocs != null) {

2
lucene/src/java/org/apache/lucene/index/SegmentMerger.java
View File

@ -308,6 +308,8 @@ final class SegmentMerger {
final MergeState.IndexReaderAndLiveDocs reader = mergeState.readers.get(i);
// nocommit -- assert that final doc count ==
// mergedDocCount from stored fields and term vectors
mergeState.docBase[i] = docBase;
final int maxDoc = reader.reader.maxDoc();
final int docCount;

24
lucene/src/java/org/apache/lucene/index/SegmentReader.java
View File

@ -62,7 +62,6 @@ public final class SegmentReader extends IndexReader {
liveDocs = null;
}
numDocs = si.docCount - si.getDelCount();
assert checkLiveCounts(false);
success = true;
} finally {
// With lock-less commits, it's entirely possible (and
@ -94,7 +93,6 @@ public final class SegmentReader extends IndexReader {
// ... but load our own deleted docs:
liveDocs = si.getCodec().liveDocsFormat().readLiveDocs(si.dir, si, context);
numDocs = si.docCount - si.getDelCount();
assert checkLiveCounts(false);
// We share core w/ parent:
parent.core.incRef();
@ -114,8 +112,6 @@ public final class SegmentReader extends IndexReader {
this.liveDocs = liveDocs;
this.numDocs = numDocs;
assert checkLiveCounts(true);
}
@Override
@ -124,26 +120,6 @@ public final class SegmentReader extends IndexReader {
return liveDocs;
}
private boolean checkLiveCounts(boolean isNRT) throws IOException {
MutableBits liveDocs = (MutableBits) this.liveDocs;
if (liveDocs != null) {
if (liveDocs.length() != si.docCount) {
throw new CorruptIndexException("document count mismatch: deleted docs count " + liveDocs.length() + " vs segment doc count " + si.docCount + " segment=" + si.name);
}
final int count = liveDocs.count();
// Verify our docCount matches:
assert numDocs == count :
"delete count mismatch: numDocs=" + numDocs + " vs MutableBits=" + (si.docCount-count);
assert isNRT || si.docCount - si.getDelCount() == count :
"si.docCount=" + si.docCount + "si.getDelCount()=" + si.getDelCount() + " recomputedCount=" + count;
}
return true;
}
@Override
protected void doClose() throws IOException {
//System.out.println("SR.close seg=" + si);

3
lucene/src/java/org/apache/lucene/util/MutableBits.java
View File

@ -19,8 +19,5 @@ package org.apache.lucene.util;
public interface MutableBits extends Bits, Cloneable {
public void clear(int bit);
// nocommit: remove this from this interface somehow? (used by DWPT infostream at least)
public int count();
public MutableBits clone();
}

87
lucene/src/test/org/apache/lucene/index/TestMixedCodecs.java Normal file
View File

@ -0,0 +1,87 @@
package org.apache.lucene.index;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.codecs.preflexrw.PreFlexRWCodec;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StringField;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.Assume;
public class TestMixedCodecs extends LuceneTestCase {
public void test() throws Exception {
Assume.assumeTrue(!(Codec.getDefault() instanceof PreFlexRWCodec));
final int NUM_DOCS = atLeast(1000);
final Directory dir = newDirectory();
RandomIndexWriter w = null;
int docsLeftInThisSegment = 0;
int docUpto = 0;
while (docUpto < NUM_DOCS) {
if (docsLeftInThisSegment == 0) {
final IndexWriterConfig iwc = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random));
if (random.nextBoolean()) {
// Make sure we aggressively mix in SimpleText
// since it has different impls for all codec
// formats...
iwc.setCodec(Codec.forName("SimpleText"));
}
if (w != null) {
w.close();
}
w = new RandomIndexWriter(random, dir, iwc);
docsLeftInThisSegment = _TestUtil.nextInt(random, 10, 100);
}
final Document doc = new Document();
doc.add(newField("id", String.valueOf(docUpto), StringField.TYPE_STORED));
w.addDocument(doc);
docUpto++;
docsLeftInThisSegment--;
}
// Random delete half the docs:
final Set<Integer> deleted = new HashSet<Integer>();
while(deleted.size() < NUM_DOCS/2) {
final Integer toDelete = random.nextInt(NUM_DOCS);
if (!deleted.contains(toDelete)) {
deleted.add(toDelete);
w.deleteDocuments(new Term("id", String.valueOf(toDelete)));
if (random.nextInt(17) == 6) {
final IndexReader r = w.getReader();
assertEquals(NUM_DOCS - deleted.size(), r.numDocs());
r.close();
}
}
}
w.close();
dir.close();
}
}