From 46ae34459ccdf0b13d54d5b33298f58bcece2ee8 Mon Sep 17 00:00:00 2001 From: Michael McCandless Date: Thu, 1 May 2014 13:32:31 +0000 Subject: [PATCH] LUCENE-5611: always abort if we hit exc in StoredFieldsWriter.start/FinishDocument git-svn-id: https://svn.apache.org/repos/asf/lucene/dev/trunk@1591657 13f79535-47bb-0310-9956-ffa450edef68 --- .../lucene40/Lucene40StoredFieldsWriter.java | 5 +- .../lucene40/Lucene40TermVectorsWriter.java | 5 +- .../lucene/index/DefaultIndexingChain.java | 61 +++++++++++-------- 3 files changed, 42 insertions(+), 29 deletions(-) diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java index bcbc891baa3..b6ebe0ddadb 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40StoredFieldsWriter.java @@ -230,13 +230,14 @@ public final class Lucene40StoredFieldsWriter extends StoredFieldsWriter { @Override public void finish(FieldInfos fis, int numDocs) { - if (HEADER_LENGTH_IDX+((long) numDocs)*8 != indexStream.getFilePointer()) + long indexFP = indexStream.getFilePointer(); + if (HEADER_LENGTH_IDX+((long) numDocs)*8 != indexFP) // This is most likely a bug in Sun JRE 1.6.0_04/_05; // we detect that the bug has struck, here, and // throw an exception to prevent the corruption from // entering the index. See LUCENE-1282 for // details. - throw new RuntimeException("fdx size mismatch: docCount is " + numDocs + " but fdx file size is " + indexStream.getFilePointer() + " file=" + indexStream.toString() + "; now aborting this merge to prevent index corruption"); + throw new RuntimeException("fdx size mismatch: docCount is " + numDocs + " but fdx file size is " + indexFP + " (wrote numDocs=" + ((indexFP-HEADER_LENGTH_IDX)/8.0) + " file=" + indexStream.toString() + "; now aborting this merge to prevent index corruption"); } @Override diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java index cc9a24573fd..0ce958133fd 100644 --- a/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java +++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene40/Lucene40TermVectorsWriter.java @@ -425,13 +425,14 @@ public final class Lucene40TermVectorsWriter extends TermVectorsWriter { @Override public void finish(FieldInfos fis, int numDocs) { - if (HEADER_LENGTH_INDEX+((long) numDocs)*16 != tvx.getFilePointer()) + long indexFP = tvx.getFilePointer(); + if (HEADER_LENGTH_INDEX+((long) numDocs)*16 != indexFP) // This is most likely a bug in Sun JRE 1.6.0_04/_05; // we detect that the bug has struck, here, and // throw an exception to prevent the corruption from // entering the index. See LUCENE-1282 for // details. - throw new RuntimeException("tvx size mismatch: mergedDocs is " + numDocs + " but tvx size is " + tvx.getFilePointer() + " file=" + tvx.toString() + "; now aborting this merge to prevent index corruption"); + throw new RuntimeException("tvx size mismatch: mergedDocs is " + numDocs + " but tvx size is " + indexFP + " (wrote numDocs=" + ((indexFP - HEADER_LENGTH_INDEX)/16.0) + " file=" + tvx.toString() + "; now aborting this merge to prevent index corruption"); } /** Close all streams. */ diff --git a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java index 5d0d6726e9a..e0a411b0f67 100644 --- a/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java +++ b/lucene/core/src/java/org/apache/lucene/index/DefaultIndexingChain.java @@ -160,9 +160,8 @@ final class DefaultIndexingChain extends DocConsumer { * stored fields. */ private void fillStoredFields(int docID) throws IOException { while (lastStoredDocID < docID) { - storedFieldsWriter.startDocument(); - lastStoredDocID++; - storedFieldsWriter.finishDocument(); + startStoredFields(); + finishStoredFields(); } } @@ -242,6 +241,35 @@ final class DefaultIndexingChain extends DocConsumer { hashMask = newHashMask; } + /** Calls StoredFieldsWriter.startDocument, aborting the + * segment if it hits any exception. */ + private void startStoredFields() throws IOException { + boolean success = false; + try { + storedFieldsWriter.startDocument(); + success = true; + } finally { + if (success == false) { + docWriter.setAborting(); + } + } + lastStoredDocID++; + } + + /** Calls StoredFieldsWriter.finishDocument, aborting the + * segment if it hits any exception. */ + private void finishStoredFields() throws IOException { + boolean success = false; + try { + storedFieldsWriter.finishDocument(); + success = true; + } finally { + if (success == false) { + docWriter.setAborting(); + } + } + } + @Override public void processDocument() throws IOException { @@ -295,10 +323,9 @@ final class DefaultIndexingChain extends DocConsumer { // Add stored fields: // TODO: if these hit exc today ->>> corrumption! fillStoredFields(docState.docID); - storedFieldsWriter.startDocument(); - lastStoredDocID++; + startStoredFields(); - // TODO: clean up this looop, its complicated because dv exceptions are non-aborting, + // TODO: clean up this loop, it's complicated because dv exceptions are non-aborting, // but storedfields are. Its also bogus that docvalues are treated as stored fields... for (StorableField field : docState.doc.storableFields()) { final String fieldName = field.name(); @@ -331,28 +358,12 @@ final class DefaultIndexingChain extends DocConsumer { } finally { if (!success) { // dv failed: so just try to bail on the current doc by calling finishDocument()... - success = false; - try { - storedFieldsWriter.finishDocument(); - success = true; - } finally { - if (!success) { - docWriter.setAborting(); - } - } + finishStoredFields(); } } } - - success = false; - try { - storedFieldsWriter.finishDocument(); - success = true; - } finally { - if (!success) { - docWriter.setAborting(); - } - } + + finishStoredFields(); } private static void verifyFieldType(String name, IndexableFieldType ft) {