From 9729b244de50322c2cc889c97c2ffb2b4675cf77 Mon Sep 17 00:00:00 2001 From: cnauroth Date: Mon, 16 Feb 2015 14:43:02 -0800 Subject: [PATCH] HDFS-7604. Track and display failed DataNode storage locations in NameNode. Contributed by Chris Nauroth. --- hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 + ...atanodeProtocolClientSideTranslatorPB.java | 9 +- ...atanodeProtocolServerSideTranslatorPB.java | 7 +- .../hadoop/hdfs/protocolPB/PBHelper.java | 25 ++ .../blockmanagement/DatanodeDescriptor.java | 39 +- .../blockmanagement/DatanodeManager.java | 7 +- .../blockmanagement/HeartbeatManager.java | 8 +- .../hdfs/server/datanode/BPServiceActor.java | 12 +- .../datanode/fsdataset/FsDatasetSpi.java | 8 + .../fsdataset/impl/FsDatasetImpl.java | 108 ++++- .../datanode/fsdataset/impl/FsVolumeList.java | 41 +- .../fsdataset/impl/VolumeFailureInfo.java | 82 ++++ .../datanode/metrics/FSDatasetMBean.java | 19 + .../hdfs/server/namenode/FSNamesystem.java | 52 ++- .../server/namenode/NameNodeRpcServer.java | 6 +- .../namenode/metrics/FSNamesystemMBean.java | 13 + .../server/protocol/DatanodeProtocol.java | 5 +- .../server/protocol/VolumeFailureSummary.java | 72 +++ .../src/main/proto/DatanodeProtocol.proto | 17 +- .../src/main/webapps/hdfs/dfshealth.html | 33 ++ .../src/main/webapps/hdfs/dfshealth.js | 43 ++ .../blockmanagement/TestBlockManager.java | 3 +- .../TestNameNodePrunesMissingStorages.java | 4 +- .../TestOverReplicatedBlocks.java | 2 +- .../TestReplicationPolicy.java | 2 +- .../TestReplicationPolicyConsiderLoad.java | 8 +- .../TestReplicationPolicyWithNodeGroup.java | 2 +- .../server/datanode/SimulatedFSDataset.java | 22 + .../server/datanode/TestBPOfferService.java | 4 +- .../server/datanode/TestBlockRecovery.java | 4 +- .../TestDataNodeVolumeFailureReporting.java | 419 ++++++++++++++++-- .../server/datanode/TestFsDatasetCache.java | 3 +- .../server/datanode/TestStorageReport.java | 4 +- .../extdataset/ExternalDatasetImpl.java | 21 + .../fsdataset/impl/TestFsDatasetImpl.java | 6 +- .../fsdataset/impl/TestFsVolumeList.java | 7 +- .../namenode/NNThroughputBenchmark.java | 4 +- .../hdfs/server/namenode/NameNodeAdapter.java | 2 +- .../server/namenode/TestDeadDatanode.java | 2 +- 39 files changed, 1022 insertions(+), 106 deletions(-) create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java create mode 100644 hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 324d5ae5b49..cc24dc4c1a0 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -634,6 +634,9 @@ Release 2.7.0 - UNRELEASED HDFS-7430. Refactor the BlockScanner to use O(1) memory and use multiple threads (cmccabe) + HDFS-7604. Track and display failed DataNode storage locations in NameNode. + (cnauroth) + OPTIMIZATIONS HDFS-7454. Reduce memory footprint for AclEntries in NameNode. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java index 46023ecaa34..192916f365d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java @@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.ipc.ProtobufHelper; import org.apache.hadoop.ipc.ProtobufRpcEngine; import org.apache.hadoop.ipc.ProtocolMetaInterface; @@ -121,8 +122,8 @@ public class DatanodeProtocolClientSideTranslatorPB implements @Override public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration, StorageReport[] reports, long cacheCapacity, long cacheUsed, - int xmitsInProgress, int xceiverCount, int failedVolumes) - throws IOException { + int xmitsInProgress, int xceiverCount, int failedVolumes, + VolumeFailureSummary volumeFailureSummary) throws IOException { HeartbeatRequestProto.Builder builder = HeartbeatRequestProto.newBuilder() .setRegistration(PBHelper.convert(registration)) .setXmitsInProgress(xmitsInProgress).setXceiverCount(xceiverCount) @@ -134,6 +135,10 @@ public class DatanodeProtocolClientSideTranslatorPB implements if (cacheUsed != 0) { builder.setCacheUsed(cacheUsed); } + if (volumeFailureSummary != null) { + builder.setVolumeFailureSummary(PBHelper.convertVolumeFailureSummary( + volumeFailureSummary)); + } HeartbeatResponseProto resp; try { resp = rpcProxy.sendHeartbeat(NULL_CONTROLLER, builder.build()); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java index d01673507b9..1a89090f13e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import com.google.protobuf.RpcController; import com.google.protobuf.ServiceException; @@ -104,10 +105,14 @@ public class DatanodeProtocolServerSideTranslatorPB implements try { final StorageReport[] report = PBHelper.convertStorageReports( request.getReportsList()); + VolumeFailureSummary volumeFailureSummary = + request.hasVolumeFailureSummary() ? PBHelper.convertVolumeFailureSummary( + request.getVolumeFailureSummary()) : null; response = impl.sendHeartbeat(PBHelper.convert(request.getRegistration()), report, request.getCacheCapacity(), request.getCacheUsed(), request.getXmitsInProgress(), - request.getXceiverCount(), request.getFailedVolumes()); + request.getXceiverCount(), request.getFailedVolumes(), + volumeFailureSummary); } catch (IOException e) { throw new ServiceException(e); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java index 8ecd767ef2b..76db47b6da9 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java @@ -122,6 +122,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.KeyUpdateCom import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.NNHAStatusHeartbeatProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto; import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto; +import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.VolumeFailureSummaryProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto; import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto; @@ -215,6 +216,7 @@ import org.apache.hadoop.hdfs.server.protocol.RegisterCommand; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog; import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId; import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId; import org.apache.hadoop.hdfs.util.ExactSizeInputStream; @@ -1901,6 +1903,29 @@ public class PBHelper { return protos; } + public static VolumeFailureSummary convertVolumeFailureSummary( + VolumeFailureSummaryProto proto) { + List failedStorageLocations = proto.getFailedStorageLocationsList(); + return new VolumeFailureSummary( + failedStorageLocations.toArray(new String[failedStorageLocations.size()]), + proto.getLastVolumeFailureDate(), proto.getEstimatedCapacityLostTotal()); + } + + public static VolumeFailureSummaryProto convertVolumeFailureSummary( + VolumeFailureSummary volumeFailureSummary) { + VolumeFailureSummaryProto.Builder builder = + VolumeFailureSummaryProto.newBuilder(); + for (String failedStorageLocation: + volumeFailureSummary.getFailedStorageLocations()) { + builder.addFailedStorageLocations(failedStorageLocation); + } + builder.setLastVolumeFailureDate( + volumeFailureSummary.getLastVolumeFailureDate()); + builder.setEstimatedCapacityLostTotal( + volumeFailureSummary.getEstimatedCapacityLostTotal()); + return builder.build(); + } + public static JournalInfo convert(JournalInfoProto info) { int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0; int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0; diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java index 833ab560f22..b8980134a88 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo; import org.apache.hadoop.hdfs.server.namenode.CachedBlock; import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.hdfs.util.EnumCounters; import org.apache.hadoop.hdfs.util.LightWeightHashSet; import org.apache.hadoop.util.IntrusiveCollection; @@ -214,6 +215,7 @@ public class DatanodeDescriptor extends DatanodeInfo { private long lastBlocksScheduledRollTime = 0; private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min private int volumeFailures = 0; + private VolumeFailureSummary volumeFailureSummary = null; /** * When set to true, the node is not in include list and is not allowed @@ -233,7 +235,7 @@ public class DatanodeDescriptor extends DatanodeInfo { */ public DatanodeDescriptor(DatanodeID nodeID) { super(nodeID); - updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0); + updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null); } /** @@ -244,7 +246,7 @@ public class DatanodeDescriptor extends DatanodeInfo { public DatanodeDescriptor(DatanodeID nodeID, String networkLocation) { super(nodeID, networkLocation); - updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0); + updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null); } @VisibleForTesting @@ -345,9 +347,10 @@ public class DatanodeDescriptor extends DatanodeInfo { * Updates stats from datanode heartbeat. */ public void updateHeartbeat(StorageReport[] reports, long cacheCapacity, - long cacheUsed, int xceiverCount, int volFailures) { + long cacheUsed, int xceiverCount, int volFailures, + VolumeFailureSummary volumeFailureSummary) { updateHeartbeatState(reports, cacheCapacity, cacheUsed, xceiverCount, - volFailures); + volFailures, volumeFailureSummary); heartbeatedSinceRegistration = true; } @@ -355,7 +358,8 @@ public class DatanodeDescriptor extends DatanodeInfo { * process datanode heartbeat or stats initialization. */ public void updateHeartbeatState(StorageReport[] reports, long cacheCapacity, - long cacheUsed, int xceiverCount, int volFailures) { + long cacheUsed, int xceiverCount, int volFailures, + VolumeFailureSummary volumeFailureSummary) { long totalCapacity = 0; long totalRemaining = 0; long totalBlockPoolUsed = 0; @@ -370,7 +374,10 @@ public class DatanodeDescriptor extends DatanodeInfo { // during the current DN registration session. // When volumeFailures == this.volumeFailures, it implies there is no // state change. No need to check for failed storage. This is an - // optimization. + // optimization. Recent versions of the DataNode report a + // VolumeFailureSummary containing the date/time of the last volume + // failure. If that's available, then we check that instead for greater + // accuracy. // 2. After DN restarts, volFailures might not increase and it is possible // we still have new failed storage. For example, admins reduce // available storages in configuration. Another corner case @@ -379,8 +386,14 @@ public class DatanodeDescriptor extends DatanodeInfo { // one element in storageReports and that is A. b) A failed. c) Before // DN sends HB to NN to indicate A has failed, DN restarts. d) After DN // restarts, storageReports has one element which is B. - boolean checkFailedStorages = (volFailures > this.volumeFailures) || - !heartbeatedSinceRegistration; + final boolean checkFailedStorages; + if (volumeFailureSummary != null && this.volumeFailureSummary != null) { + checkFailedStorages = volumeFailureSummary.getLastVolumeFailureDate() > + this.volumeFailureSummary.getLastVolumeFailureDate(); + } else { + checkFailedStorages = (volFailures > this.volumeFailures) || + !heartbeatedSinceRegistration; + } if (checkFailedStorages) { LOG.info("Number of failed storage changes from " @@ -394,6 +407,7 @@ public class DatanodeDescriptor extends DatanodeInfo { setXceiverCount(xceiverCount); setLastUpdate(Time.now()); this.volumeFailures = volFailures; + this.volumeFailureSummary = volumeFailureSummary; for (StorageReport report : reports) { DatanodeStorageInfo storage = updateStorage(report.getStorage()); if (checkFailedStorages) { @@ -728,6 +742,15 @@ public class DatanodeDescriptor extends DatanodeInfo { return volumeFailures; } + /** + * Returns info about volume failures. + * + * @return info about volume failures, possibly null + */ + public VolumeFailureSummary getVolumeFailureSummary() { + return volumeFailureSummary; + } + /** * @param nodeReg DatanodeID to update registration for. */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java index f5fe1613655..45c56a8b5fd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java @@ -1387,8 +1387,8 @@ public class DatanodeManager { public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg, StorageReport[] reports, final String blockPoolId, long cacheCapacity, long cacheUsed, int xceiverCount, - int maxTransfers, int failedVolumes - ) throws IOException { + int maxTransfers, int failedVolumes, + VolumeFailureSummary volumeFailureSummary) throws IOException { synchronized (heartbeatManager) { synchronized (datanodeMap) { DatanodeDescriptor nodeinfo = null; @@ -1410,7 +1410,8 @@ public class DatanodeManager { heartbeatManager.updateHeartbeat(nodeinfo, reports, cacheCapacity, cacheUsed, - xceiverCount, failedVolumes); + xceiverCount, failedVolumes, + volumeFailureSummary); // If we are in safemode, do not send back any recovery / replication // requests. Don't even drain the existing queue of work. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index 9286609399d..d60a39b277d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.DFSUtil; import org.apache.hadoop.hdfs.protocol.DatanodeID; import org.apache.hadoop.hdfs.server.namenode.Namesystem; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.util.Daemon; import org.apache.hadoop.util.Time; @@ -192,7 +193,7 @@ class HeartbeatManager implements DatanodeStatistics { addDatanode(d); //update its timestamp - d.updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0); + d.updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null); } } @@ -217,10 +218,11 @@ class HeartbeatManager implements DatanodeStatistics { synchronized void updateHeartbeat(final DatanodeDescriptor node, StorageReport[] reports, long cacheCapacity, long cacheUsed, - int xceiverCount, int failedVolumes) { + int xceiverCount, int failedVolumes, + VolumeFailureSummary volumeFailureSummary) { stats.subtract(node); node.updateHeartbeat(reports, cacheCapacity, cacheUsed, - xceiverCount, failedVolumes); + xceiverCount, failedVolumes, volumeFailureSummary); stats.add(node); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java index 69210db8edf..ff1ad786d1d 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java @@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.ipc.RemoteException; import org.apache.hadoop.util.Time; @@ -579,14 +580,19 @@ class BPServiceActor implements Runnable { LOG.debug("Sending heartbeat with " + reports.length + " storage reports from service actor: " + this); } - + + VolumeFailureSummary volumeFailureSummary = dn.getFSDataset() + .getVolumeFailureSummary(); + int numFailedVolumes = volumeFailureSummary != null ? + volumeFailureSummary.getFailedStorageLocations().length : 0; return bpNamenode.sendHeartbeat(bpRegistration, reports, dn.getFSDataset().getCacheCapacity(), dn.getFSDataset().getCacheUsed(), dn.getXmitsInProgress(), dn.getXceiverCount(), - dn.getFSDataset().getNumFailedVolumes()); + numFailedVolumes, + volumeFailureSummary); } //This must be called only by BPOfferService @@ -1019,4 +1025,4 @@ class BPServiceActor implements Runnable { } } } -} \ No newline at end of file +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java index 7d880d3bd3a..437b795cc15 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java @@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.util.DiskChecker.DiskErrorException; import org.apache.hadoop.util.ReflectionUtils; @@ -129,6 +130,13 @@ public interface FsDatasetSpi extends FSDatasetMBean { /** @return a volume information map (name => info). */ public Map getVolumeInfoMap(); + /** + * Returns info about volume failures. + * + * @return info about volume failures, possibly null + */ + VolumeFailureSummary getVolumeFailureSummary(); + /** @return a list of finalized blocks for the given block pool. */ public List getFinalizedBlocks(String bpid); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java index 8c8d744ae0c..2b9feb536a5 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java @@ -99,6 +99,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.MultipleIOException; import org.apache.hadoop.io.nativeio.NativeIO; @@ -112,6 +113,7 @@ import org.apache.hadoop.util.Time; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; +import com.google.common.collect.Sets; /************************************************** * FSDataset manages a set of data blocks. Each block @@ -264,9 +266,11 @@ class FsDatasetImpl implements FsDatasetSpi { String[] dataDirs = conf.getTrimmedStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY); Collection dataLocations = DataNode.getStorageLocations(conf); + List volumeFailureInfos = getInitialVolumeFailureInfos( + dataLocations, storage); int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length; - int volsFailed = volsConfigured - storage.getNumStorageDirs(); + int volsFailed = volumeFailureInfos.size(); this.validVolsRequired = volsConfigured - volFailuresTolerated; if (volFailuresTolerated < 0 || volFailuresTolerated >= volsConfigured) { @@ -291,7 +295,7 @@ class FsDatasetImpl implements FsDatasetSpi { DFSConfigKeys.DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY, RoundRobinVolumeChoosingPolicy.class, VolumeChoosingPolicy.class), conf); - volumes = new FsVolumeList(volsFailed, datanode.getBlockScanner(), + volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(), blockChooserImpl); asyncDiskService = new FsDatasetAsyncDiskService(datanode); asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode); @@ -313,6 +317,36 @@ class FsDatasetImpl implements FsDatasetSpi { DFSConfigKeys.DFS_DATANODE_BLOCK_PINNING_ENABLED_DEFAULT); } + /** + * Gets initial volume failure information for all volumes that failed + * immediately at startup. The method works by determining the set difference + * between all configured storage locations and the actual storage locations in + * use after attempting to put all of them into service. + * + * @return each storage location that has failed + */ + private static List getInitialVolumeFailureInfos( + Collection dataLocations, DataStorage storage) { + Set failedLocationSet = Sets.newHashSetWithExpectedSize( + dataLocations.size()); + for (StorageLocation sl: dataLocations) { + failedLocationSet.add(sl.getFile().getAbsolutePath()); + } + for (Iterator it = storage.dirIterator(); + it.hasNext(); ) { + Storage.StorageDirectory sd = it.next(); + failedLocationSet.remove(sd.getRoot().getAbsolutePath()); + } + List volumeFailureInfos = Lists.newArrayListWithCapacity( + failedLocationSet.size()); + long failureDate = Time.now(); + for (String failedStorageLocation: failedLocationSet) { + volumeFailureInfos.add(new VolumeFailureInfo(failedStorageLocation, + failureDate)); + } + return volumeFailureInfos; + } + private void addVolume(Collection dataLocations, Storage.StorageDirectory sd) throws IOException { final File dir = sd.getCurrentDir(); @@ -348,8 +382,14 @@ class FsDatasetImpl implements FsDatasetSpi { final File dir = location.getFile(); // Prepare volume in DataStorage - DataStorage.VolumeBuilder builder = - dataStorage.prepareVolume(datanode, location.getFile(), nsInfos); + final DataStorage.VolumeBuilder builder; + try { + builder = dataStorage.prepareVolume(datanode, location.getFile(), nsInfos); + } catch (IOException e) { + volumes.addVolumeFailureInfo(new VolumeFailureInfo( + location.getFile().getAbsolutePath(), Time.now())); + throw e; + } final Storage.StorageDirectory sd = builder.getStorageDirectory(); @@ -498,9 +538,65 @@ class FsDatasetImpl implements FsDatasetSpi { /** * Return the number of failed volumes in the FSDataset. */ - @Override + @Override // FSDatasetMBean public int getNumFailedVolumes() { - return volumes.numberOfFailedVolumes(); + return volumes.getVolumeFailureInfos().length; + } + + @Override // FSDatasetMBean + public String[] getFailedStorageLocations() { + VolumeFailureInfo[] infos = volumes.getVolumeFailureInfos(); + List failedStorageLocations = Lists.newArrayListWithCapacity( + infos.length); + for (VolumeFailureInfo info: infos) { + failedStorageLocations.add(info.getFailedStorageLocation()); + } + return failedStorageLocations.toArray( + new String[failedStorageLocations.size()]); + } + + @Override // FSDatasetMBean + public long getLastVolumeFailureDate() { + long lastVolumeFailureDate = 0; + for (VolumeFailureInfo info: volumes.getVolumeFailureInfos()) { + long failureDate = info.getFailureDate(); + if (failureDate > lastVolumeFailureDate) { + lastVolumeFailureDate = failureDate; + } + } + return lastVolumeFailureDate; + } + + @Override // FSDatasetMBean + public long getEstimatedCapacityLostTotal() { + long estimatedCapacityLostTotal = 0; + for (VolumeFailureInfo info: volumes.getVolumeFailureInfos()) { + estimatedCapacityLostTotal += info.getEstimatedCapacityLost(); + } + return estimatedCapacityLostTotal; + } + + @Override // FsDatasetSpi + public VolumeFailureSummary getVolumeFailureSummary() { + VolumeFailureInfo[] infos = volumes.getVolumeFailureInfos(); + if (infos.length == 0) { + return null; + } + List failedStorageLocations = Lists.newArrayListWithCapacity( + infos.length); + long lastVolumeFailureDate = 0; + long estimatedCapacityLostTotal = 0; + for (VolumeFailureInfo info: infos) { + failedStorageLocations.add(info.getFailedStorageLocation()); + long failureDate = info.getFailureDate(); + if (failureDate > lastVolumeFailureDate) { + lastVolumeFailureDate = failureDate; + } + estimatedCapacityLostTotal += info.getEstimatedCapacityLost(); + } + return new VolumeFailureSummary( + failedStorageLocations.toArray(new String[failedStorageLocations.size()]), + lastVolumeFailureDate, estimatedCapacityLostTotal); } @Override // FSDatasetMBean diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java index ae2f5b4cba7..4573172c699 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java @@ -22,9 +22,12 @@ import java.io.IOException; import java.nio.channels.ClosedChannelException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.Collections; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.atomic.AtomicReference; import com.google.common.collect.Lists; @@ -40,21 +43,23 @@ import org.apache.hadoop.util.Time; class FsVolumeList { private final AtomicReference volumes = new AtomicReference<>(new FsVolumeImpl[0]); + // Tracks volume failures, sorted by volume path. + private final Map volumeFailureInfos = + Collections.synchronizedMap(new TreeMap()); private Object checkDirsMutex = new Object(); private final VolumeChoosingPolicy blockChooser; private final BlockScanner blockScanner; - private volatile int numFailedVolumes; - FsVolumeList(int failedVols, BlockScanner blockScanner, + FsVolumeList(List initialVolumeFailureInfos, + BlockScanner blockScanner, VolumeChoosingPolicy blockChooser) { this.blockChooser = blockChooser; this.blockScanner = blockScanner; - this.numFailedVolumes = failedVols; - } - - int numberOfFailedVolumes() { - return numFailedVolumes; + for (VolumeFailureInfo volumeFailureInfo: initialVolumeFailureInfos) { + volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(), + volumeFailureInfo); + } } /** @@ -238,7 +243,7 @@ class FsVolumeList { } removedVols.add(fsv); removeVolume(fsv); - numFailedVolumes++; + addVolumeFailureInfo(fsv); } catch (ClosedChannelException e) { FsDatasetImpl.LOG.debug("Caught exception when obtaining " + "reference count on closed volume", e); @@ -347,6 +352,26 @@ class FsVolumeList { removeVolume(fsVolume); } } + removeVolumeFailureInfo(volume); + } + + VolumeFailureInfo[] getVolumeFailureInfos() { + Collection infos = volumeFailureInfos.values(); + return infos.toArray(new VolumeFailureInfo[infos.size()]); + } + + void addVolumeFailureInfo(VolumeFailureInfo volumeFailureInfo) { + volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(), + volumeFailureInfo); + } + + private void addVolumeFailureInfo(FsVolumeImpl vol) { + addVolumeFailureInfo(new VolumeFailureInfo(vol.getBasePath(), Time.now(), + vol.getCapacity())); + } + + private void removeVolumeFailureInfo(File vol) { + volumeFailureInfos.remove(vol.getAbsolutePath()); } void addBlockPool(final String bpid, final Configuration conf) throws IOException { diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java new file mode 100644 index 00000000000..c3ce2a4632f --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java @@ -0,0 +1,82 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl; + +/** + * Tracks information about failure of a data volume. + */ +final class VolumeFailureInfo { + private final String failedStorageLocation; + private final long failureDate; + private final long estimatedCapacityLost; + + /** + * Creates a new VolumeFailureInfo, when the capacity lost from this volume + * failure is unknown. Typically, this means the volume failed immediately at + * startup, so there was never a chance to query its capacity. + * + * @param failedStorageLocation storage location that has failed + * @param failureDate date/time of failure in milliseconds since epoch + */ + public VolumeFailureInfo(String failedStorageLocation, long failureDate) { + this(failedStorageLocation, failureDate, 0); + } + + /** + * Creates a new VolumeFailureInfo. + * + * @param failedStorageLocation storage location that has failed + * @param failureDate date/time of failure in milliseconds since epoch + * @param estimatedCapacityLost estimate of capacity lost in bytes + */ + public VolumeFailureInfo(String failedStorageLocation, long failureDate, + long estimatedCapacityLost) { + this.failedStorageLocation = failedStorageLocation; + this.failureDate = failureDate; + this.estimatedCapacityLost = estimatedCapacityLost; + } + + /** + * Returns the storage location that has failed. + * + * @return storage location that has failed + */ + public String getFailedStorageLocation() { + return this.failedStorageLocation; + } + + /** + * Returns date/time of failure + * + * @return date/time of failure in milliseconds since epoch + */ + public long getFailureDate() { + return this.failureDate; + } + + /** + * Returns estimate of capacity lost. This is said to be an estimate, because + * in some cases it's impossible to know the capacity of the volume, such as if + * we never had a chance to query its capacity before the failure occurred. + * + * @return estimate of capacity lost in bytes + */ + public long getEstimatedCapacityLost() { + return this.estimatedCapacityLost; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java index 8388c0b38df..5f22540afdf 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java @@ -78,6 +78,25 @@ public interface FSDatasetMBean { */ public int getNumFailedVolumes(); + /** + * Returns each storage location that has failed, sorted. + * @return each storage location that has failed, sorted + */ + String[] getFailedStorageLocations(); + + /** + * Returns the date/time of the last volume failure in milliseconds since + * epoch. + * @return date/time of last volume failure in milliseconds since epoch + */ + long getLastVolumeFailureDate(); + + /** + * Returns an estimate of total capacity lost due to volume failures in bytes. + * @return estimate of total capacity lost in bytes + */ + long getEstimatedCapacityLostTotal(); + /** * Returns the amount of cache used by the datanode (in bytes). */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 5fd1efbd73c..84ab179790e 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -250,6 +250,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration; import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.IOUtils; @@ -4412,8 +4413,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, */ HeartbeatResponse handleHeartbeat(DatanodeRegistration nodeReg, StorageReport[] reports, long cacheCapacity, long cacheUsed, - int xceiverCount, int xmitsInProgress, int failedVolumes) - throws IOException { + int xceiverCount, int xmitsInProgress, int failedVolumes, + VolumeFailureSummary volumeFailureSummary) throws IOException { readLock(); try { //get datanode commands @@ -4421,7 +4422,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, - xmitsInProgress; DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat( nodeReg, reports, blockPoolId, cacheCapacity, cacheUsed, - xceiverCount, maxTransfer, failedVolumes); + xceiverCount, maxTransfer, failedVolumes, volumeFailureSummary); //create ha status final NNHAStatusHeartbeat haState = new NNHAStatusHeartbeat( @@ -5942,6 +5943,32 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, return deadDecommissioned; } + @Override // FSNamesystemMBean + public int getVolumeFailuresTotal() { + List live = new ArrayList(); + getBlockManager().getDatanodeManager().fetchDatanodes(live, null, true); + int volumeFailuresTotal = 0; + for (DatanodeDescriptor node: live) { + volumeFailuresTotal += node.getVolumeFailures(); + } + return volumeFailuresTotal; + } + + @Override // FSNamesystemMBean + public long getEstimatedCapacityLostTotal() { + List live = new ArrayList(); + getBlockManager().getDatanodeManager().fetchDatanodes(live, null, true); + long estimatedCapacityLostTotal = 0; + for (DatanodeDescriptor node: live) { + VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary(); + if (volumeFailureSummary != null) { + estimatedCapacityLostTotal += + volumeFailureSummary.getEstimatedCapacityLostTotal(); + } + } + return estimatedCapacityLostTotal; + } + @Override // FSNamesystemMBean public int getNumDecommissioningDataNodes() { return getBlockManager().getDatanodeManager().getDecommissioningNodes() @@ -6785,7 +6812,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, final List live = new ArrayList(); blockManager.getDatanodeManager().fetchDatanodes(live, null, true); for (DatanodeDescriptor node : live) { - Map innerinfo = ImmutableMap.builder() + ImmutableMap.Builder innerinfo = + ImmutableMap.builder(); + innerinfo .put("infoAddr", node.getInfoAddr()) .put("infoSecureAddr", node.getInfoSecureAddr()) .put("xferaddr", node.getXferAddr()) @@ -6801,9 +6830,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, .put("blockScheduled", node.getBlocksScheduled()) .put("blockPoolUsed", node.getBlockPoolUsed()) .put("blockPoolUsedPercent", node.getBlockPoolUsedPercent()) - .put("volfails", node.getVolumeFailures()) - .build(); - info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo); + .put("volfails", node.getVolumeFailures()); + VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary(); + if (volumeFailureSummary != null) { + innerinfo + .put("failedStorageLocations", + volumeFailureSummary.getFailedStorageLocations()) + .put("lastVolumeFailureDate", + volumeFailureSummary.getLastVolumeFailureDate()) + .put("estimatedCapacityLostTotal", + volumeFailureSummary.getEstimatedCapacityLostTotal()); + } + info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo.build()); } return JSON.toString(info); } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java index 54f5f8566e8..da34bd25242 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java @@ -136,6 +136,7 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest; import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport; import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks; import org.apache.hadoop.hdfs.server.protocol.StorageReport; +import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary; import org.apache.hadoop.hdfs.StorageType; import org.apache.hadoop.io.EnumSetWritable; import org.apache.hadoop.io.Text; @@ -1281,12 +1282,13 @@ class NameNodeRpcServer implements NamenodeProtocols { public HeartbeatResponse sendHeartbeat(DatanodeRegistration nodeReg, StorageReport[] report, long dnCacheCapacity, long dnCacheUsed, int xmitsInProgress, int xceiverCount, - int failedVolumes) throws IOException { + int failedVolumes, VolumeFailureSummary volumeFailureSummary) + throws IOException { checkNNStartup(); verifyRequest(nodeReg); return namesystem.handleHeartbeat(nodeReg, report, dnCacheCapacity, dnCacheUsed, xceiverCount, xmitsInProgress, - failedVolumes); + failedVolumes, volumeFailureSummary); } @Override // DatanodeProtocol diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java index 86f4bd624ec..b31b7b6b722 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java @@ -131,6 +131,19 @@ public interface FSNamesystemMBean { */ public int getNumDecomDeadDataNodes(); + /** + * Number of failed data volumes across all live data nodes. + * @return number of failed data volumes across all live data nodes + */ + int getVolumeFailuresTotal(); + + /** + * Returns an estimate of total capacity lost due to volume failures in bytes + * across all live data nodes. + * @return estimate of total capacity lost in bytes + */ + long getEstimatedCapacityLostTotal(); + /** * Number of data nodes that are in the decommissioning state */ diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java index c3b0f68ec83..047de569618 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java @@ -102,6 +102,7 @@ public interface DatanodeProtocol { * @param xmitsInProgress number of transfers from this datanode to others * @param xceiverCount number of active transceiver threads * @param failedVolumes number of failed volumes + * @param volumeFailureSummary info about volume failures * @throws IOException on error */ @Idempotent @@ -111,7 +112,9 @@ public interface DatanodeProtocol { long dnCacheUsed, int xmitsInProgress, int xceiverCount, - int failedVolumes) throws IOException; + int failedVolumes, + VolumeFailureSummary volumeFailureSummary) + throws IOException; /** * blockReport() tells the NameNode about all the locally-stored blocks. diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java new file mode 100644 index 00000000000..1722dd0e55d --- /dev/null +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java @@ -0,0 +1,72 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hdfs.server.protocol; + +/** + * Summarizes information about data volume failures on a DataNode. + */ +public class VolumeFailureSummary { + private final String[] failedStorageLocations; + private final long lastVolumeFailureDate; + private final long estimatedCapacityLostTotal; + + /** + * Creates a new VolumeFailureSummary. + * + * @param failedStorageLocations storage locations that have failed + * @param lastVolumeFailureDate date/time of last volume failure in + * milliseconds since epoch + * @param estimatedCapacityLostTotal estimate of capacity lost in bytes + */ + public VolumeFailureSummary(String[] failedStorageLocations, + long lastVolumeFailureDate, long estimatedCapacityLostTotal) { + this.failedStorageLocations = failedStorageLocations; + this.lastVolumeFailureDate = lastVolumeFailureDate; + this.estimatedCapacityLostTotal = estimatedCapacityLostTotal; + } + + /** + * Returns each storage location that has failed, sorted. + * + * @return each storage location that has failed, sorted + */ + public String[] getFailedStorageLocations() { + return this.failedStorageLocations; + } + + /** + * Returns the date/time of the last volume failure in milliseconds since + * epoch. + * + * @return date/time of last volume failure in milliseconds since epoch + */ + public long getLastVolumeFailureDate() { + return this.lastVolumeFailureDate; + } + + /** + * Returns estimate of capacity lost. This is said to be an estimate, because + * in some cases it's impossible to know the capacity of the volume, such as if + * we never had a chance to query its capacity before the failure occurred. + * + * @return estimate of capacity lost in bytes + */ + public long getEstimatedCapacityLostTotal() { + return this.estimatedCapacityLostTotal; + } +} diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto index 187761a4502..348f34606e2 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto @@ -160,6 +160,17 @@ message RegisterDatanodeResponseProto { required DatanodeRegistrationProto registration = 1; // Datanode info } +/** + * failedStorageLocations - storage locations that have failed + * lastVolumeFailureDate - date/time of last volume failure + * estimatedCapacityLost - estimate of total capacity lost due to volume failures + */ +message VolumeFailureSummaryProto { + repeated string failedStorageLocations = 1; + required uint64 lastVolumeFailureDate = 2; + required uint64 estimatedCapacityLostTotal = 3; +} + /** * registration - datanode registration information * capacity - total storage capacity available at the datanode @@ -168,9 +179,12 @@ message RegisterDatanodeResponseProto { * blockPoolUsed - storage used by the block pool * xmitsInProgress - number of transfers from this datanode to others * xceiverCount - number of active transceiver threads - * failedVolumes - number of failed volumes + * failedVolumes - number of failed volumes. This is redundant with the + * information included in volumeFailureSummary, but the field is retained + * for backwards compatibility. * cacheCapacity - total cache capacity available at the datanode * cacheUsed - amount of cache used + * volumeFailureSummary - info about volume failures */ message HeartbeatRequestProto { required DatanodeRegistrationProto registration = 1; // Datanode info @@ -180,6 +194,7 @@ message HeartbeatRequestProto { optional uint32 failedVolumes = 5 [ default = 0 ]; optional uint64 cacheCapacity = 6 [ default = 0 ]; optional uint64 cacheUsed = 7 [default = 0 ]; + optional VolumeFailureSummaryProto volumeFailureSummary = 8; } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html index 9c83f3a0192..391ca79ce67 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html @@ -34,6 +34,7 @@