Apache
/
hadoop
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

HDFS-14726. Fix JN incompatibility issue in branch-2 due to backport of HDFS-10519. Contributed by Chen Liang.

This commit is contained in:
Chen Liang 2019-08-30 15:55:35 -07:00
parent 655154cb45
commit f05d87b32e
3 changed files with 24 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
View File

@ -76,6 +76,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsServerProtos.StorageUuidsProto;
import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto; import org.apache.hadoop.hdfs.protocol.proto.JournalProtocolProtos.JournalInfoProto;
import org.apache.hadoop.hdfs.security.token.block.BlockKey; import org.apache.hadoop.hdfs.security.token.block.BlockKey;
import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys; import org.apache.hadoop.hdfs.security.token.block.ExportedBlockKeys;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState; import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.ReplicaState;
@ -292,8 +293,13 @@ public class PBHelper {
for (RemoteEditLogProto l : manifest.getLogsList()) { for (RemoteEditLogProto l : manifest.getLogsList()) {
logs.add(convert(l)); logs.add(convert(l));
} }
return new RemoteEditLogManifest(logs, long committedId = HdfsServerConstants.INVALID_TXID;
manifest.getCommittedTxnId()); if (manifest.hasCommittedTxnId()) {
// An older version JN may not have this field, in which case committedId
// is set to INVALID_TXID.
committedId = manifest.getCommittedTxnId();
}
return new RemoteEditLogManifest(logs, committedId);
} }
public static CheckpointCommandProto convert(CheckpointCommand cmd) { public static CheckpointCommandProto convert(CheckpointCommand cmd) {

20
hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java
View File

@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.GetJourna
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.NewEpochResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.PrepareRecoveryResponseProto;
import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto; import org.apache.hadoop.hdfs.qjournal.protocol.QJournalProtocolProtos.SegmentStateProto;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.common.Storage; import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo; import org.apache.hadoop.hdfs.server.common.StorageInfo;
import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream; import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
@ -632,13 +633,22 @@ public class QuorumJournalManager implements JournalManager {
// If it's bounded by durable Txns, endTxId could not be larger // If it's bounded by durable Txns, endTxId could not be larger
// than committedTxnId. This ensures the consistency. // than committedTxnId. This ensures the consistency.
// An older version JN may not return the field committedTxnId,
// in which case it is set to INVALID_TXID.
if (onlyDurableTxns && inProgressOk) { if (onlyDurableTxns && inProgressOk) {
endTxId = Math.min(endTxId, committedTxnId); if (committedTxnId == HdfsServerConstants.INVALID_TXID) {
if (endTxId < remoteLog.getStartTxId()) { LOG.warn("Received undefined committed txn id, "
LOG.warn("Found endTxId (" + endTxId + ") that is less than " + + " NN and JN are on different version? "
"the startTxId (" + remoteLog.getStartTxId() + + "- seting to startTxId");
") - setting it to startTxId.");
endTxId = remoteLog.getStartTxId(); endTxId = remoteLog.getStartTxId();
} else {
endTxId = Math.min(endTxId, committedTxnId);
if (endTxId < remoteLog.getStartTxId()) {
LOG.warn("Found endTxId (" + endTxId + ") that is less than " +
"the startTxId (" + remoteLog.getStartTxId() +
") - setting it to startTxId.");
endTxId = remoteLog.getStartTxId();
}
} }
} }

2
hadoop-hdfs-project/hadoop-hdfs/src/main/proto/HdfsServer.proto
View File

@ -92,7 +92,7 @@ message RemoteEditLogProto {
*/ */
message RemoteEditLogManifestProto { message RemoteEditLogManifestProto {
repeated RemoteEditLogProto logs = 1; repeated RemoteEditLogProto logs = 1;
required uint64 committedTxnId = 2; optional uint64 committedTxnId = 2;
} }
/** /**