From abf763c1c50d085e21fb1bd4ddde6d8f1ba53895 Mon Sep 17 00:00:00 2001 From: Simon Willnauer Date: Wed, 5 Aug 2015 12:28:47 +0200 Subject: [PATCH] Rethrow exception during recovery finalization even if source if not broken Today we miss to throw / rethrow an recovery exception if it happens during the finalization of phase 1 if the source files are not affected. Even worse this can cause some dataloss if the reason for this exception is a failure of deleting a corruption marker or similar pre-existing corruptions since we continue with the recovery and mark the target shared as started which will in-turn open an engine with an empty index. --- .../indices/recovery/RecoverySourceHandler.java | 1 + .../elasticsearch/indices/recovery/RecoveryTarget.java | 8 ++++++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java index 572b784093e..295ab49ac7f 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoverySourceHandler.java @@ -435,6 +435,7 @@ public class RecoverySourceHandler { exception.addSuppressed(remoteException); logger.warn("{} Remote file corruption during finalization on node {}, recovering {}. local checksum OK", corruptIndexException, shard.shardId(), request.targetNode()); + throw exception; } else { throw remoteException; } diff --git a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java index 0388265e64c..a953206fa89 100644 --- a/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java +++ b/core/src/main/java/org/elasticsearch/indices/recovery/RecoveryTarget.java @@ -406,9 +406,13 @@ public class RecoveryTarget extends AbstractComponent { logger.debug("Failed to clean lucene index", e); ex.addSuppressed(e); } - throw new RecoveryFailedException(recoveryStatus.state(), "failed to clean after recovery", ex); + RecoveryFailedException rfe = new RecoveryFailedException(recoveryStatus.state(), "failed to clean after recovery", ex); + recoveryStatus.fail(rfe, true); + throw rfe; } catch (Exception ex) { - throw new RecoveryFailedException(recoveryStatus.state(), "failed to clean after recovery", ex); + RecoveryFailedException rfe = new RecoveryFailedException(recoveryStatus.state(), "failed to clean after recovery", ex); + recoveryStatus.fail(rfe, true); + throw rfe; } channel.sendResponse(TransportResponse.Empty.INSTANCE); }