From da61953a0bcaca4e8fafa9c985b6a860ce14f782 Mon Sep 17 00:00:00 2001 From: Eric Yang Date: Sun, 24 Jan 2016 14:19:49 -0800 Subject: [PATCH] HDFS-9653. Added blocks pending deletion report to dfsadmin. (Weiwei Yang via eyang) --- .../org/apache/hadoop/hdfs/DFSClient.java | 34 +++++++++++++------ .../hadoop/hdfs/DistributedFileSystem.java | 9 +++++ .../hadoop/hdfs/protocol/ClientProtocol.java | 4 ++- .../hdfs/protocolPB/PBHelperClient.java | 7 ++++ .../main/proto/ClientNamenodeProtocol.proto | 1 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++ .../blockmanagement/HeartbeatManager.java | 1 + .../hdfs/server/namenode/FSNamesystem.java | 2 ++ .../apache/hadoop/hdfs/tools/DFSAdmin.java | 2 ++ .../TestPendingInvalidateBlock.java | 19 +++++++++++ 10 files changed, 71 insertions(+), 11 deletions(-) diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java index ec1a6703ce8..b2dafbfa6bd 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java @@ -2018,10 +2018,11 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, } } - private long[] callGetStats() throws IOException { + private long getStateByIndex(int stateIndex) throws IOException { checkOpen(); try (TraceScope ignored = tracer.newScope("getStats")) { - return namenode.getStats(); + long[] states = namenode.getStats(); + return states.length > stateIndex ? states[stateIndex] : -1; } } @@ -2029,8 +2030,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, * @see ClientProtocol#getStats() */ public FsStatus getDiskStatus() throws IOException { - long rawNums[] = callGetStats(); - return new FsStatus(rawNums[0], rawNums[1], rawNums[2]); + return new FsStatus(getStateByIndex(0), + getStateByIndex(1), getStateByIndex(2)); } /** @@ -2039,7 +2040,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, * @throws IOException */ public long getMissingBlocksCount() throws IOException { - return callGetStats()[ClientProtocol.GET_STATS_MISSING_BLOCKS_IDX]; + return getStateByIndex(ClientProtocol. + GET_STATS_MISSING_BLOCKS_IDX); } /** @@ -2048,8 +2050,17 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, * @throws IOException */ public long getMissingReplOneBlocksCount() throws IOException { - return callGetStats()[ClientProtocol. - GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX]; + return getStateByIndex(ClientProtocol. + GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX); + } + + /** + * Returns count of blocks pending on deletion. + * @throws IOException + */ + public long getPendingDeletionBlocksCount() throws IOException { + return getStateByIndex(ClientProtocol. + GET_STATS_PENDING_DELETION_BLOCKS_IDX); } /** @@ -2057,7 +2068,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, * @throws IOException */ public long getUnderReplicatedBlocksCount() throws IOException { - return callGetStats()[ClientProtocol.GET_STATS_UNDER_REPLICATED_IDX]; + return getStateByIndex(ClientProtocol. + GET_STATS_UNDER_REPLICATED_IDX); } /** @@ -2065,7 +2077,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, * @throws IOException */ public long getCorruptBlocksCount() throws IOException { - return callGetStats()[ClientProtocol.GET_STATS_CORRUPT_BLOCKS_IDX]; + return getStateByIndex(ClientProtocol. + GET_STATS_CORRUPT_BLOCKS_IDX); } /** @@ -2075,7 +2088,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory, * @throws IOException */ public long getBytesInFutureBlocks() throws IOException { - return callGetStats()[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX]; + return getStateByIndex(ClientProtocol. + GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX); } /** diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java index 32e700d9490..7d63b9d5b5f 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java @@ -1194,6 +1194,15 @@ public class DistributedFileSystem extends FileSystem { return dfs.getMissingBlocksCount(); } + /** + * Returns count of blocks pending on deletion. + * + * @throws IOException + */ + public long getPendingDeletionBlocksCount() throws IOException { + return dfs.getPendingDeletionBlocksCount(); + } + /** * Returns count of blocks with replication factor 1 and have * lost the only replica. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java index dcf7c7bb632..5fa80257699 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java @@ -714,7 +714,8 @@ public interface ClientProtocol { int GET_STATS_MISSING_BLOCKS_IDX = 5; int GET_STATS_MISSING_REPL_ONE_BLOCKS_IDX = 6; int GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX = 7; - int STATS_ARRAY_LENGTH = 8; + int GET_STATS_PENDING_DELETION_BLOCKS_IDX = 8; + int STATS_ARRAY_LENGTH = 9; /** * Get a set of statistics about the filesystem. @@ -729,6 +730,7 @@ public interface ClientProtocol { *
  • [6] contains number of blocks which have replication factor * 1 and have lost the only replica.
    • *
  • [7] contains number of bytes that are at risk for deletion.
    • + *
  • [8] contains number of pending deletion blocks.
    • * * Use public constants like {@link #GET_STATS_CAPACITY_IDX} in place of * actual numbers to index into the array. diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java index ff6f8925d38..5d81c1acbe1 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java @@ -1509,6 +1509,8 @@ public class PBHelperClient { res.getMissingReplOneBlocks(); result[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX] = res.hasBlocksInFuture() ? res.getBlocksInFuture() : 0; + result[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX] = + res.getPendingDeletionBlocks(); return result; } @@ -1869,6 +1871,11 @@ public class PBHelperClient { result.setBlocksInFuture( fsStats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX]); } + if (fsStats.length >= + ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX + 1) { + result.setPendingDeletionBlocks( + fsStats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX]); + } return result.build(); } diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto index 075229725cd..9925d1d3bbe 100644 --- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto +++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto @@ -306,6 +306,7 @@ message GetFsStatsResponseProto { required uint64 missing_blocks = 6; optional uint64 missing_repl_one_blocks = 7; optional uint64 blocks_in_future = 8; + optional uint64 pending_deletion_blocks = 9; } enum DatanodeReportTypeProto { // type of the datanode report diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt index 8ffe1c75b59..8d33532e4b8 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt +++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt @@ -82,6 +82,9 @@ Release 2.8.0 - UNRELEASED IMPROVEMENTS + HDFS-9653. Added blocks pending deletion report to dfsadmin. + (Weiwei Yang via eyang) + HDFS-9257. improve error message for "Absolute path required" in INode.java to contain the rejected path (Marcell Szabo via harsh) diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java index 7546b1ac3ab..b8d30437729 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java @@ -184,6 +184,7 @@ class HeartbeatManager implements DatanodeStatistics { -1L, -1L, -1L, + -1L, -1L}; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java index 9a44776ece4..225d55941a4 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java @@ -3772,6 +3772,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean, getMissingReplOneBlocksCount(); stats[ClientProtocol.GET_STATS_BYTES_IN_FUTURE_BLOCKS_IDX] = blockManager.getBytesInFuture(); + stats[ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX] = + blockManager.getPendingDeletionBlocksCount(); return stats; } diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java index fe2a497fd9e..59fe1a7910c 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java @@ -503,6 +503,8 @@ public class DFSAdmin extends FsShell { dfs.getMissingBlocksCount()); System.out.println("Missing blocks (with replication factor 1): " + dfs.getMissingReplOneBlocksCount()); + System.out.println("Pending deletion blocks: " + + dfs.getPendingDeletionBlocksCount()); System.out.println(); diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java index a588a73150e..75005440319 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestPendingInvalidateBlock.java @@ -17,16 +17,19 @@ */ package org.apache.hadoop.hdfs.server.blockmanagement; +import java.lang.reflect.Method; import java.text.SimpleDateFormat; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeys; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hdfs.DFSClient; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DFSTestUtil; import org.apache.hadoop.hdfs.DistributedFileSystem; import org.apache.hadoop.hdfs.MiniDFSCluster; import org.apache.hadoop.hdfs.MiniDFSCluster.DataNodeProperties; +import org.apache.hadoop.hdfs.protocol.ClientProtocol; import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils; import org.apache.hadoop.test.GenericTestUtils; import org.apache.log4j.Level; @@ -88,15 +91,31 @@ public class TestPendingInvalidateBlock { Assert.assertEquals(0, cluster.getNamesystem().getBlocksTotal()); Assert.assertEquals(REPLICATION, cluster.getNamesystem() .getPendingDeletionBlocks()); + Assert.assertEquals(REPLICATION, + dfs.getPendingDeletionBlocksCount()); Thread.sleep(6000); Assert.assertEquals(0, cluster.getNamesystem().getBlocksTotal()); Assert.assertEquals(0, cluster.getNamesystem().getPendingDeletionBlocks()); + Assert.assertEquals(0, dfs.getPendingDeletionBlocksCount()); long nnStarted = cluster.getNamesystem().getNNStartedTimeInMillis(); long blockDeletionStartTime = cluster.getNamesystem() .getBlockDeletionStartTime(); Assert.assertTrue(String.format( "Expect blockDeletionStartTime = %d > nnStarted = %d.", blockDeletionStartTime, nnStarted), blockDeletionStartTime > nnStarted); + + // test client protocol compatibility + Method method = DFSClient.class. + getDeclaredMethod("getStateByIndex", int.class); + method.setAccessible(true); + // get number of pending deletion blocks by its index + long validState = (Long) method.invoke(dfs.getClient(), + ClientProtocol.GET_STATS_PENDING_DELETION_BLOCKS_IDX); + // get an out of index value + long invalidState = (Long) method.invoke(dfs.getClient(), + ClientProtocol.STATS_ARRAY_LENGTH); + Assert.assertEquals(0, validState); + Assert.assertEquals(-1, invalidState); } /**