From 59841b887220a64a2273a98e0fc23078564b1314 Mon Sep 17 00:00:00 2001 From: Justin Bertram Date: Fri, 21 Jul 2017 19:38:33 -0500 Subject: [PATCH] ARTEMIS-1302 make quorum voting more transparent --- .../impl/wireformat/QuorumVoteMessage.java | 16 +++++++ .../core/server/ActiveMQServerLogger.java | 48 +++++++++++++++++++ .../core/server/cluster/ClusterControl.java | 5 +- .../server/cluster/ClusterController.java | 2 + .../server/cluster/qourum/BooleanVote.java | 6 ++- .../server/cluster/qourum/QuorumManager.java | 2 + .../qourum/QuorumVoteServerConnect.java | 11 +++-- .../cluster/qourum/ServerConnectVote.java | 5 ++ .../qourum/SharedNothingBackupQuorum.java | 2 + .../core/server/impl/ColocatedActivation.java | 5 ++ .../server/impl/ServerConnectVoteHandler.java | 3 ++ 11 files changed, 100 insertions(+), 5 deletions(-) diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/wireformat/QuorumVoteMessage.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/wireformat/QuorumVoteMessage.java index 5c030aefa6..2f4e0248ed 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/wireformat/QuorumVoteMessage.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/protocol/core/impl/wireformat/QuorumVoteMessage.java @@ -67,4 +67,20 @@ public class QuorumVoteMessage extends PacketImpl { public void decode(QuorumVoteHandler voteHandler) { vote = voteHandler.decode(voteBuffer); } + + + @Override + public String toString() { + StringBuffer buff = new StringBuffer(getParentString()); + buff.append("]"); + return buff.toString(); + } + + @Override + public String getParentString() { + StringBuffer buff = new StringBuffer(super.getParentString()); + buff.append(", vote=" + vote); + buff.append(", handler=" + handler); + return buff.toString(); + } } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java index 431302fb4c..f930f02f26 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/ActiveMQServerLogger.java @@ -346,6 +346,54 @@ public interface ActiveMQServerLogger extends BasicLogger { @Message(id = 221059, value = "Deleting old data directory {0} as the max folders is set to 0", format = Message.Format.MESSAGE_FORMAT) void backupDeletingData(String oldPath); + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221060, value = "Sending quorum vote request to {0}: {1}", format = Message.Format.MESSAGE_FORMAT) + void sendingQuorumVoteRequest(String remoteAddress, String vote); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221061, value = "Received quorum vote response from {0}: {1}", format = Message.Format.MESSAGE_FORMAT) + void receivedQuorumVoteResponse(String remoteAddress, String vote); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221062, value = "Received quorum vote request: {0}", format = Message.Format.MESSAGE_FORMAT) + void receivedQuorumVoteRequest(String vote); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221063, value = "Sending quorum vote response: {0}", format = Message.Format.MESSAGE_FORMAT) + void sendingQuorumVoteResponse(String vote); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221064, value = "Node {0} found in cluster topology", format = Message.Format.MESSAGE_FORMAT) + void nodeFoundInClusterTopology(String nodeId); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221065, value = "Node {0} not found in cluster topology", format = Message.Format.MESSAGE_FORMAT) + void nodeNotFoundInClusterTopology(String nodeId); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221066, value = "Initiating quorum vote: {0}", format = Message.Format.MESSAGE_FORMAT) + void initiatingQuorumVote(SimpleString vote); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221067, value = "Waiting {0} {1} for quorum vote results.", format = Message.Format.MESSAGE_FORMAT) + void waitingForQuorumVoteResults(int timeout, String unit); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221068, value = "Received all quorum votes.", format = Message.Format.MESSAGE_FORMAT) + void receivedAllQuorumVotes(); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221069, value = "Timeout waiting for quorum vote responses.", format = Message.Format.MESSAGE_FORMAT) + void timeoutWaitingForQuorumVoteResponses(); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221070, value = "Restarting as backup based on quorum vote results.", format = Message.Format.MESSAGE_FORMAT) + void restartingAsBackupBasedOnQuorumVoteResults(); + + @LogMessage(level = Logger.Level.INFO) + @Message(id = 221071, value = "Failing over based on quorum vote results.", format = Message.Format.MESSAGE_FORMAT) + void failingOverBasedOnQuorumVoteResults(); + @LogMessage(level = Logger.Level.WARN) @Message(id = 222000, value = "ActiveMQServer is being finalized and has not been stopped. Please remember to stop the server before letting it go out of scope", format = Message.Format.MESSAGE_FORMAT) diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterControl.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterControl.java index b37b9d5139..07f0fc283c 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterControl.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterControl.java @@ -159,10 +159,13 @@ public class ClusterControl implements AutoCloseable { public Vote sendQuorumVote(SimpleString handler, Vote vote) { try { + ActiveMQServerLogger.LOGGER.sendingQuorumVoteRequest(getSessionFactory().getConnection().getRemoteAddress(), vote.toString()); QuorumVoteReplyMessage replyMessage = (QuorumVoteReplyMessage) clusterChannel.sendBlocking(new QuorumVoteMessage(handler, vote), PacketImpl.QUORUM_VOTE_REPLY); QuorumVoteHandler voteHandler = server.getClusterManager().getQuorumManager().getVoteHandler(replyMessage.getHandler()); replyMessage.decodeRest(voteHandler); - return replyMessage.getVote(); + Vote voteResponse = replyMessage.getVote(); + ActiveMQServerLogger.LOGGER.receivedQuorumVoteResponse(getSessionFactory().getConnection().getRemoteAddress(), voteResponse.toString()); + return voteResponse; } catch (ActiveMQException e) { return null; } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterController.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterController.java index b0d0232690..464e945675 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterController.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/ClusterController.java @@ -379,7 +379,9 @@ public class ClusterController implements ActiveMQComponent { QuorumVoteMessage quorumVoteMessage = (QuorumVoteMessage) packet; QuorumVoteHandler voteHandler = quorumManager.getVoteHandler(quorumVoteMessage.getHandler()); quorumVoteMessage.decode(voteHandler); + ActiveMQServerLogger.LOGGER.receivedQuorumVoteRequest(quorumVoteMessage.getVote().toString()); Vote vote = quorumManager.vote(quorumVoteMessage.getHandler(), quorumVoteMessage.getVote()); + ActiveMQServerLogger.LOGGER.sendingQuorumVoteResponse(vote.toString()); clusterChannel.send(new QuorumVoteReplyMessage(quorumVoteMessage.getHandler(), vote)); } else if (packet.getType() == PacketImpl.SCALEDOWN_ANNOUNCEMENT) { ScaleDownAnnounceMessage message = (ScaleDownAnnounceMessage) packet; diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/BooleanVote.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/BooleanVote.java index 5d615660c1..90f55ed498 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/BooleanVote.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/BooleanVote.java @@ -25,7 +25,7 @@ import org.apache.activemq.artemis.api.core.ActiveMQBuffer; */ public class BooleanVote extends Vote { - private boolean vote; + protected boolean vote; public BooleanVote(boolean vote) { this.vote = vote; @@ -56,4 +56,8 @@ public class BooleanVote extends Vote { vote = buff.readBoolean(); } + @Override + public String toString() { + return "BooleanVote [vote=" + vote + "]"; + } } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumManager.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumManager.java index dad772b8d7..f8b3908151 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumManager.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumManager.java @@ -29,6 +29,7 @@ import org.apache.activemq.artemis.api.core.client.ClusterTopologyListener; import org.apache.activemq.artemis.api.core.client.TopologyMember; import org.apache.activemq.artemis.core.client.impl.TopologyMemberImpl; import org.apache.activemq.artemis.core.server.ActiveMQComponent; +import org.apache.activemq.artemis.core.server.ActiveMQServerLogger; import org.apache.activemq.artemis.core.server.cluster.ClusterControl; import org.apache.activemq.artemis.core.server.cluster.ClusterController; @@ -181,6 +182,7 @@ public final class QuorumManager implements ClusterTopologyListener, ActiveMQCom if (!started) return; //send a vote to each node + ActiveMQServerLogger.LOGGER.initiatingQuorumVote(quorumVote.getName()); for (TopologyMemberImpl tm : clusterController.getDefaultClusterTopology().getMembers()) { //but not ourselves if (!tm.getNodeId().equals(clusterController.getNodeID().toString())) { diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumVoteServerConnect.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumVoteServerConnect.java index a189155dba..f6c608ef96 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumVoteServerConnect.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/cluster/qourum/QuorumVoteServerConnect.java @@ -21,13 +21,14 @@ import java.util.concurrent.TimeUnit; import org.apache.activemq.artemis.api.core.SimpleString; import org.apache.activemq.artemis.core.client.impl.Topology; +import org.apache.activemq.artemis.core.server.ActiveMQServerLogger; /** - * A Qourum Vote for deciding if a replicated backup should become live. + * A Quorum Vote for deciding if a replicated backup should become live. */ public class QuorumVoteServerConnect extends QuorumVote { - public static final SimpleString LIVE_FAILOVER_VOTE = new SimpleString("LIVE_FAILOVER_VOTE"); + public static final SimpleString LIVE_FAILOVER_VOTE = new SimpleString("LiveFailoverQuorumVote"); private final CountDownLatch latch; private final String targetNodeId; @@ -123,6 +124,10 @@ public class QuorumVoteServerConnect extends QuorumVote getVote() { return new Pair<>(nodeID, backupsSize); } + + @Override + public String toString() { + return "RequestBackupVote [backupsSize=" + backupsSize + ", nodeID=" + nodeID + ", backupAvailable=" + backupAvailable + "]"; + } } } diff --git a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ServerConnectVoteHandler.java b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ServerConnectVoteHandler.java index 84603573ed..ef0438a500 100644 --- a/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ServerConnectVoteHandler.java +++ b/artemis-server/src/main/java/org/apache/activemq/artemis/core/server/impl/ServerConnectVoteHandler.java @@ -19,6 +19,7 @@ package org.apache.activemq.artemis.core.server.impl; import org.apache.activemq.artemis.api.core.ActiveMQBuffer; import org.apache.activemq.artemis.api.core.SimpleString; import org.apache.activemq.artemis.core.client.impl.TopologyMemberImpl; +import org.apache.activemq.artemis.core.server.ActiveMQServerLogger; import org.apache.activemq.artemis.core.server.cluster.qourum.QuorumVoteHandler; import org.apache.activemq.artemis.core.server.cluster.qourum.QuorumVoteServerConnect; import org.apache.activemq.artemis.core.server.cluster.qourum.ServerConnectVote; @@ -37,8 +38,10 @@ public class ServerConnectVoteHandler implements QuorumVoteHandler { String nodeid = serverConnectVote.getNodeId(); TopologyMemberImpl member = server.getClusterManager().getDefaultConnection(null).getTopology().getMember(nodeid); if (member != null && member.getLive() != null) { + ActiveMQServerLogger.LOGGER.nodeFoundInClusterTopology(nodeid); return new ServerConnectVote(nodeid, false); } + ActiveMQServerLogger.LOGGER.nodeNotFoundInClusterTopology(nodeid); return new ServerConnectVote(nodeid, true); }