Apache
/
hbase
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HBASE-24360 RollingBatchRestartRsAction loses track of dead servers 

`RollingBatchRestartRsAction` doesn't handle failure cases when
tracking its list of dead servers. The original author believed that a
failure to restart would result in a retry. However, by removing the
dead server from the failed list, that state is lost, and retry never
occurs. Because this action doesn't ever look back to the current
state of the cluster, relying only on its local state for the current
action invocation, it never realizes the abandoned server is still
dead. Instead, be more careful to only remove the dead server from the
list when the `startRs` invocation claims to have been successful.

Signed-off-by: stack <stack@apache.org>
This commit is contained in:
Nick Dimiduk 2020-05-18 12:08:52 -07:00 committed by Nick Dimiduk
parent 696894e2cc
commit 0dae377f53
2 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/Action.java
View File

@ -190,7 +190,7 @@ public abstract class Action {
getLogger().info("Stopping regionserver {}", server); getLogger().info("Stopping regionserver {}", server);
cluster.stopRegionServer(server); cluster.stopRegionServer(server);
cluster.waitForRegionServerToStop(server, killRsTimeout); cluster.waitForRegionServerToStop(server, killRsTimeout);
getLogger().info("Stoppiong regionserver {}. Reported num of rs:{}", server, getLogger().info("Stopping regionserver {}. Reported num of rs:{}", server,
cluster.getClusterMetrics().getLiveServerMetrics().size()); cluster.getClusterMetrics().getLiveServerMetrics().size());
} }

14
hbase-it/src/test/java/org/apache/hadoop/hbase/chaos/actions/RollingBatchRestartRsAction.java
View File

@ -20,8 +20,10 @@ package org.apache.hadoop.hbase.chaos.actions;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Collections;
import java.util.LinkedList; import java.util.LinkedList;
import java.util.List; import java.util.List;
import java.util.Objects;
import java.util.Queue; import java.util.Queue;
import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.RandomUtils;
import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.ServerName;
@ -70,7 +72,7 @@ public class RollingBatchRestartRsAction extends BatchRestartRsAction {
List<ServerName> selectedServers = selectServers(); List<ServerName> selectedServers = selectServers();
Queue<ServerName> serversToBeKilled = new LinkedList<>(selectedServers); Queue<ServerName> serversToBeKilled = new LinkedList<>(selectedServers);
Queue<ServerName> deadServers = new LinkedList<>(); LinkedList<ServerName> deadServers = new LinkedList<>();
// loop while there are servers to be killed or dead servers to be restarted // loop while there are servers to be killed or dead servers to be restarted
while ((!serversToBeKilled.isEmpty() || !deadServers.isEmpty()) && !context.isStopping()) { while ((!serversToBeKilled.isEmpty() || !deadServers.isEmpty()) && !context.isStopping()) {
@ -103,13 +105,17 @@ public class RollingBatchRestartRsAction extends BatchRestartRsAction {
deadServers.add(server); deadServers.add(server);
break; break;
case START: case START:
server = Objects.requireNonNull(deadServers.peek());
try { try {
server = deadServers.remove();
startRs(server); startRs(server);
// only remove the server from the known dead list if `startRs` succeeds.
deadServers.remove(server);
} catch (org.apache.hadoop.util.Shell.ExitCodeException e) { } catch (org.apache.hadoop.util.Shell.ExitCodeException e) {
// The start may fail but better to just keep going though we may lose server. // The start may fail but better to just keep going though we may lose server.
// // Shuffle the dead list to avoid getting stuck on a single stubborn host.
getLogger().info("Problem starting, will retry; code={}", e.getExitCode(), e); Collections.shuffle(deadServers);
getLogger().info(
"Problem starting {}, will retry; code={}", server, e.getExitCode(), e);
} }
break; break;
} }