cutover RandomIndexWriter to return sequence numbers

This commit is contained in:
Mike McCandless 2016-05-24 20:22:47 -04:00
parent 67c35aedb9
commit 69c28eac11
6 changed files with 73 additions and 48 deletions

View File

@ -161,11 +161,15 @@ final class DocumentsWriter implements Closeable, Accountable {
return seqNo; return seqNo;
} }
synchronized boolean updateDocValues(DocValuesUpdate... updates) throws IOException { synchronized long updateDocValues(DocValuesUpdate... updates) throws IOException {
final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue; final DocumentsWriterDeleteQueue deleteQueue = this.deleteQueue;
deleteQueue.addDocValuesUpdates(updates); long seqNo = deleteQueue.addDocValuesUpdates(updates);
flushControl.doOnDelete(); flushControl.doOnDelete();
return applyAllDeletes(deleteQueue); if (applyAllDeletes(deleteQueue)) {
seqNo = -seqNo;
}
return seqNo;
} }
DocumentsWriterDeleteQueue currentDeleteSession() { DocumentsWriterDeleteQueue currentDeleteSession() {

View File

@ -1571,17 +1571,23 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* @throws IOException * @throws IOException
* if there is a low-level IO error * if there is a low-level IO error
*/ */
public void updateNumericDocValue(Term term, String field, long value) throws IOException { public long updateNumericDocValue(Term term, String field, long value) throws IOException {
ensureOpen(); ensureOpen();
if (!globalFieldNumberMap.contains(field, DocValuesType.NUMERIC)) { if (!globalFieldNumberMap.contains(field, DocValuesType.NUMERIC)) {
throw new IllegalArgumentException("can only update existing numeric-docvalues fields!"); throw new IllegalArgumentException("can only update existing numeric-docvalues fields!");
} }
try { try {
if (docWriter.updateDocValues(new NumericDocValuesUpdate(term, field, value))) { long seqNo = docWriter.updateDocValues(new NumericDocValuesUpdate(term, field, value));
if (seqNo < 0) {
seqNo = -seqNo;
processEvents(true, false); processEvents(true, false);
} }
return seqNo;
} catch (VirtualMachineError tragedy) { } catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "updateNumericDocValue"); tragicEvent(tragedy, "updateNumericDocValue");
// dead code but javac disagrees:
return -1;
} }
} }
@ -1605,7 +1611,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* @throws IOException * @throws IOException
* if there is a low-level IO error * if there is a low-level IO error
*/ */
public void updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException { public long updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
ensureOpen(); ensureOpen();
if (value == null) { if (value == null) {
throw new IllegalArgumentException("cannot update a field to a null value: " + field); throw new IllegalArgumentException("cannot update a field to a null value: " + field);
@ -1614,11 +1620,17 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
throw new IllegalArgumentException("can only update existing binary-docvalues fields!"); throw new IllegalArgumentException("can only update existing binary-docvalues fields!");
} }
try { try {
if (docWriter.updateDocValues(new BinaryDocValuesUpdate(term, field, value))) { long seqNo = docWriter.updateDocValues(new BinaryDocValuesUpdate(term, field, value));
if (seqNo < 0) {
seqNo = -seqNo;
processEvents(true, false); processEvents(true, false);
} }
return seqNo;
} catch (VirtualMachineError tragedy) { } catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "updateBinaryDocValue"); tragicEvent(tragedy, "updateBinaryDocValue");
// dead code but javac disagrees:
return -1;
} }
} }
@ -1635,7 +1647,7 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
* @throws IOException * @throws IOException
* if there is a low-level IO error * if there is a low-level IO error
*/ */
public void updateDocValues(Term term, Field... updates) throws IOException { public long updateDocValues(Term term, Field... updates) throws IOException {
ensureOpen(); ensureOpen();
DocValuesUpdate[] dvUpdates = new DocValuesUpdate[updates.length]; DocValuesUpdate[] dvUpdates = new DocValuesUpdate[updates.length];
for (int i = 0; i < updates.length; i++) { for (int i = 0; i < updates.length; i++) {
@ -1662,11 +1674,17 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
} }
} }
try { try {
if (docWriter.updateDocValues(dvUpdates)) { long seqNo = docWriter.updateDocValues(dvUpdates);
if (seqNo < 0) {
seqNo = -seqNo;
processEvents(true, false); processEvents(true, false);
} }
return seqNo;
} catch (VirtualMachineError tragedy) { } catch (VirtualMachineError tragedy) {
tragicEvent(tragedy, "updateDocValues"); tragicEvent(tragedy, "updateDocValues");
// dead code but javac disagrees:
return -1;
} }
} }
@ -3045,7 +3063,6 @@ public class IndexWriter implements Closeable, TwoPhaseCommit, Accountable {
@Override @Override
public final long commit() throws IOException { public final long commit() throws IOException {
ensureOpen(); ensureOpen();
// nocommit should we put seq no into sis?
return commitInternal(config.getMergePolicy()); return commitInternal(config.getMergePolicy());
} }

View File

@ -50,7 +50,5 @@ public interface TwoPhaseCommit {
* {@link #prepareCommit()}, this method is used to roll all other objects * {@link #prepareCommit()}, this method is used to roll all other objects
* back to their previous state. * back to their previous state.
*/ */
// nocommit return long?
public void rollback() throws IOException; public void rollback() throws IOException;
} }

View File

@ -50,8 +50,7 @@ public class TestTwoPhaseCommitTool extends LuceneTestCase {
if (failOnPrepare) { if (failOnPrepare) {
throw new IOException("failOnPrepare"); throw new IOException("failOnPrepare");
} }
// nocommit hmm return 1;
return -1;
} }
@Override @Override
@ -65,8 +64,7 @@ public class TestTwoPhaseCommitTool extends LuceneTestCase {
if (failOnCommit) { if (failOnCommit) {
throw new RuntimeException("failOnCommit"); throw new RuntimeException("failOnCommit");
} }
// nocommit hmm return 1;
return -1;
} }
@Override @Override

View File

@ -581,14 +581,14 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter {
} }
@Override @Override
public synchronized void commit() throws IOException { public synchronized long commit() throws IOException {
ensureOpen(); ensureOpen();
// LUCENE-4972: if we always call setCommitData, we create empty commits // LUCENE-4972: if we always call setCommitData, we create empty commits
String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH); String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH);
if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) { if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) {
indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData())); indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
} }
indexWriter.commit(); return indexWriter.commit();
} }
/** Combine original user data with the taxonomy epoch. */ /** Combine original user data with the taxonomy epoch. */
@ -616,14 +616,14 @@ public class DirectoryTaxonomyWriter implements TaxonomyWriter {
* See {@link IndexWriter#prepareCommit}. * See {@link IndexWriter#prepareCommit}.
*/ */
@Override @Override
public synchronized void prepareCommit() throws IOException { public synchronized long prepareCommit() throws IOException {
ensureOpen(); ensureOpen();
// LUCENE-4972: if we always call setCommitData, we create empty commits // LUCENE-4972: if we always call setCommitData, we create empty commits
String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH); String epochStr = indexWriter.getCommitData().get(INDEX_EPOCH);
if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) { if (epochStr == null || Long.parseLong(epochStr, 16) != indexEpoch) {
indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData())); indexWriter.setCommitData(combinedCommitData(indexWriter.getCommitData()));
} }
indexWriter.prepareCommit(); return indexWriter.prepareCommit();
} }
@Override @Override

View File

@ -130,14 +130,15 @@ public class RandomIndexWriter implements Closeable {
* Adds a Document. * Adds a Document.
* @see IndexWriter#addDocument(Iterable) * @see IndexWriter#addDocument(Iterable)
*/ */
public <T extends IndexableField> void addDocument(final Iterable<T> doc) throws IOException { public <T extends IndexableField> long addDocument(final Iterable<T> doc) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
long seqNo;
if (r.nextInt(5) == 3) { if (r.nextInt(5) == 3) {
// TODO: maybe, we should simply buffer up added docs // TODO: maybe, we should simply buffer up added docs
// (but we need to clone them), and only when // (but we need to clone them), and only when
// getReader, commit, etc. are called, we do an // getReader, commit, etc. are called, we do an
// addDocuments? Would be better testing. // addDocuments? Would be better testing.
w.addDocuments(new Iterable<Iterable<T>>() { seqNo = w.addDocuments(new Iterable<Iterable<T>>() {
@Override @Override
public Iterator<Iterable<T>> iterator() { public Iterator<Iterable<T>> iterator() {
@ -167,10 +168,12 @@ public class RandomIndexWriter implements Closeable {
} }
}); });
} else { } else {
w.addDocument(doc); seqNo = w.addDocument(doc);
} }
maybeFlushOrCommit(); maybeFlushOrCommit();
return seqNo;
} }
private void maybeFlushOrCommit() throws IOException { private void maybeFlushOrCommit() throws IOException {
@ -195,26 +198,29 @@ public class RandomIndexWriter implements Closeable {
} }
} }
public void addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException { public long addDocuments(Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.addDocuments(docs); long seqNo = w.addDocuments(docs);
maybeFlushOrCommit(); maybeFlushOrCommit();
return seqNo;
} }
public void updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException { public long updateDocuments(Term delTerm, Iterable<? extends Iterable<? extends IndexableField>> docs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.updateDocuments(delTerm, docs); long seqNo = w.updateDocuments(delTerm, docs);
maybeFlushOrCommit(); maybeFlushOrCommit();
return seqNo;
} }
/** /**
* Updates a document. * Updates a document.
* @see IndexWriter#updateDocument(Term, Iterable) * @see IndexWriter#updateDocument(Term, Iterable)
*/ */
public <T extends IndexableField> void updateDocument(Term t, final Iterable<T> doc) throws IOException { public <T extends IndexableField> long updateDocument(Term t, final Iterable<T> doc) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
long seqNo;
if (r.nextInt(5) == 3) { if (r.nextInt(5) == 3) {
w.updateDocuments(t, new Iterable<Iterable<T>>() { seqNo = w.updateDocuments(t, new Iterable<Iterable<T>>() {
@Override @Override
public Iterator<Iterable<T>> iterator() { public Iterator<Iterable<T>> iterator() {
@ -243,49 +249,51 @@ public class RandomIndexWriter implements Closeable {
} }
}); });
} else { } else {
w.updateDocument(t, doc); seqNo = w.updateDocument(t, doc);
} }
maybeFlushOrCommit(); maybeFlushOrCommit();
return seqNo;
} }
public void addIndexes(Directory... dirs) throws IOException { public long addIndexes(Directory... dirs) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.addIndexes(dirs); return w.addIndexes(dirs);
} }
public void addIndexes(CodecReader... readers) throws IOException { public long addIndexes(CodecReader... readers) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.addIndexes(readers); return w.addIndexes(readers);
} }
public void updateNumericDocValue(Term term, String field, Long value) throws IOException { public long updateNumericDocValue(Term term, String field, Long value) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.updateNumericDocValue(term, field, value); return w.updateNumericDocValue(term, field, value);
} }
public void updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException { public long updateBinaryDocValue(Term term, String field, BytesRef value) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.updateBinaryDocValue(term, field, value); return w.updateBinaryDocValue(term, field, value);
} }
public void updateDocValues(Term term, Field... updates) throws IOException { public long updateDocValues(Term term, Field... updates) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.updateDocValues(term, updates); return w.updateDocValues(term, updates);
} }
public void deleteDocuments(Term term) throws IOException { public long deleteDocuments(Term term) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.deleteDocuments(term); return w.deleteDocuments(term);
} }
public void deleteDocuments(Query q) throws IOException { public long deleteDocuments(Query q) throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.deleteDocuments(q); return w.deleteDocuments(q);
} }
public void commit() throws IOException { public long commit() throws IOException {
LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig()); LuceneTestCase.maybeChangeLiveIndexWriterConfig(r, w.getConfig());
w.commit(); return w.commit();
} }
public int numDocs() { public int numDocs() {
@ -296,8 +304,8 @@ public class RandomIndexWriter implements Closeable {
return w.maxDoc(); return w.maxDoc();
} }
public void deleteAll() throws IOException { public long deleteAll() throws IOException {
w.deleteAll(); return w.deleteAll();
} }
public DirectoryReader getReader() throws IOException { public DirectoryReader getReader() throws IOException {